diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 9be72d3d5..c482ae643 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -404,7 +404,7 @@ steps: - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test" timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -m "full_model and cuda" --run-level "full_model" + - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_1 - label: ":full_moon: Diffusion X2V · Doc Test" diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 70f7662ba..585a22bdc 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -43,7 +43,7 @@ th { | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | | `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | -| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | +| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | | | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index 5df7f99cd..c8f7fe617 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -33,6 +33,25 @@ This folder provides a unified CLI script for image-to-video generation using vL Default model: `Wan-AI/Wan2.2-I2V-A14B-Diffusers`. +### LingBot-Video TI2V + +LingBot-Video accepts exactly one first-frame image and uses the same dense or +MoE checkpoint as its T2I and T2V modes: + +```bash +python examples/offline_inference/image_to_video/image_to_video.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --model-class-name LingBotVideoPipeline \ + --image /path/to/input.png \ + --prompt "the fox looks toward the camera" \ + --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \ + --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \ + --output lingbot_ti2v.mp4 +``` + +The input image keeps its original geometry until the LingBot pipeline applies +its model-specific resize and center crop. + ## Prerequisites Download the example image used in the snippets below: diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index 6ebc7ae63..4485e21c3 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -43,6 +43,23 @@ This folder provides several entrypoints for experimenting with text-to-image di Default model: `Qwen/Qwen-Image` +### LingBot-Video + +LingBot-Video uses the same checkpoint for image and video generation. Run the +shared T2I entry point with: + +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --prompt "a red fox standing in fresh snow" \ + --height 192 --width 320 --num-inference-steps 2 \ + --guidance-scale 3.0 --extra-body '{"flow_shift":3.0}' \ + --output lingbot_t2i.png +``` + +The LingBot prompt builder selects the `image` output modality and requests one +frame. Passing an input image is not supported in LingBot T2I mode. + ## Quick Start ### Python API diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index 0df1f9bd0..abc522db8 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -55,6 +55,21 @@ python text_to_video.py \ --output vace_t2v_output.mp4 ``` +### LingBot-Video + +The dedicated LingBot example explicitly selects the video output modality: + +```bash +python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \ + --output lingbot_t2v.mp4 +``` + +Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE +checkpoint. Requested frame counts are rounded upward to the causal +VAE's `4n+1` grid. + ### LTX-2 ```bash diff --git a/examples/offline_inference/text_to_video/text_to_video_lingbot.py b/examples/offline_inference/text_to_video/text_to_video_lingbot.py index a8f8a2015..108e2dc1f 100644 --- a/examples/offline_inference/text_to_video/text_to_video_lingbot.py +++ b/examples/offline_inference/text_to_video/text_to_video_lingbot.py @@ -20,9 +20,9 @@ from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.") - parser.add_argument("--model", default="/home/models/lingbot-video-dense-1.3b") + parser.add_argument("--model", default="robbyant/lingbot-video-dense-1.3b") parser.add_argument("--prompt", default="a robotic arm picks up a red block") - parser.add_argument("--negative-prompt", default="") + parser.add_argument("--negative-prompt", default=None) parser.add_argument("--output", default="lingbot_video_output.mp4") parser.add_argument("--height", type=int, default=192) parser.add_argument("--width", type=int, default=320) @@ -58,9 +58,12 @@ def _to_frame_array(video: Any) -> np.ndarray: if isinstance(video, torch.Tensor): tensor = video.detach().cpu() if tensor.dim() == 5: - tensor = tensor[0] if tensor.shape[1] not in (3, 4) else tensor[0].permute(1, 2, 3, 0) - elif tensor.dim() == 4 and tensor.shape[0] in (3, 4): - tensor = tensor.permute(1, 2, 3, 0) + tensor = tensor[0] + if tensor.dim() == 4 and tensor.shape[-1] not in (3, 4): + if tensor.shape[0] in (3, 4): + tensor = tensor.permute(1, 2, 3, 0) + elif tensor.shape[1] in (3, 4): + tensor = tensor.permute(0, 2, 3, 1) if tensor.is_floating_point(): if float(tensor.min()) < 0.0: tensor = tensor.clamp(-1, 1) * 0.5 + 0.5 @@ -87,8 +90,12 @@ def main() -> None: parallel_config=DiffusionParallelConfig(), ) - prompt: dict[str, str] = {"prompt": args.prompt} - if args.negative_prompt: + prompt: dict[str, Any] = { + "prompt": args.prompt, + "modalities": ["video"], + "num_frames": args.num_frames, + } + if args.negative_prompt is not None: prompt["negative_prompt"] = args.negative_prompt sampling_params = OmniDiffusionSamplingParams( diff --git a/examples/online_serving/lingbot_video/README.md b/examples/online_serving/lingbot_video/README.md new file mode 100644 index 000000000..1c50b06de --- /dev/null +++ b/examples/online_serving/lingbot_video/README.md @@ -0,0 +1,74 @@ +# LingBot-Video + +LingBot-Video uses one `LingBotVideoPipeline` for text-to-image (T2I), +text-to-video (T2V), and text-image-to-video (TI2V) generation. Both the dense +and MoE checkpoints use the same request format. + +## Start the server + +```bash +MODEL=robbyant/lingbot-video-dense-1.3b bash run_server.sh +``` + +The MoE checkpoint uses the same server and request scripts, but requires +substantially more GPU memory: + +```bash +MODEL=robbyant/lingbot-video-moe-30b-a3b bash run_server.sh +``` + +## Text to image + +The image endpoint selects T2I mode and always generates one frame: + +```bash +bash run_curl_text_to_image.sh +``` + +The script sends a `320x192`, two-step smoke request and writes +`lingbot_t2i.png`. + +## Text or text-image to video + +Run the video script without an image to select T2V mode: + +```bash +bash run_curl_text_image_to_video.sh +``` + +Pass a first-frame image to the same script to select TI2V mode: + +```bash +INPUT_IMAGE=/path/to/input.png bash run_curl_text_image_to_video.sh +``` + +The client scripts omit the optional `model` request field, so they target +whichever dense or MoE checkpoint the server loaded. The video example uses the +lightweight `320x192`, 9-frame, two-step configuration. + +Until the shared `/v1/videos` reference-image resizing is removed, TI2V target +dimensions must be sent through `extra_params`, for example +`{"size":"320x192"}`. Do not use the top-level `size`, `width`, or `height` +fields for TI2V because the serving layer currently applies those dimensions +to the reference image before the model receives it. T2V requests can continue +to use the top-level dimension fields. + +LingBot video frame counts use the causal VAE `4n+1` grid. The pipeline rounds +any requested frame count upward to the next valid value. An explicit +`num_frames` takes precedence over `seconds`; otherwise, the server first +resolves `seconds * fps` and the pipeline applies the same alignment. + +Official `resolution`/`ratio` presets can be sent through `extra_params`, for +example `{"resolution":"720p","ratio":"16:9"}`. The `2k` and `4k` entries +only define output dimensions; whether they run successfully depends on the +checkpoint, GPU memory, and memory optimizations available in the deployment. + +For `/v1/images/generations`, the server resolves these aliases to their final +pixel dimensions before applying `--max-generated-image-size`. Requests above +the configured limit return HTTP 400 before engine dispatch. LingBot produces +one output per prompt; image requests with `n>1` are also rejected with HTTP +400. + +LingBot TI2V accepts exactly one image reference. Image editing, video +references, audio references, batching, and Refiner execution are not supported +by this pipeline mode. diff --git a/examples/online_serving/lingbot_video/run_curl_text_image_to_video.sh b/examples/online_serving/lingbot_video/run_curl_text_image_to_video.sh new file mode 100755 index 000000000..b4a9647cb --- /dev/null +++ b/examples/online_serving/lingbot_video/run_curl_text_image_to_video.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -euo pipefail + +if [ -n "${INPUT_IMAGE:-}" ] && [ ! -f "${INPUT_IMAGE}" ]; then + echo "INPUT_IMAGE does not exist: ${INPUT_IMAGE}" + exit 1 +fi + +BASE_URL="${BASE_URL:-http://localhost:8099}" +POLL_INTERVAL="${POLL_INTERVAL:-2}" + +if [ -n "${INPUT_IMAGE:-}" ]; then + default_output="lingbot_ti2v.mp4" +else + default_output="lingbot_t2v.mp4" +fi +OUTPUT_PATH="${OUTPUT_PATH:-${default_output}}" + +curl_args=( + -sS -X POST "${BASE_URL}/v1/videos" + -H "Accept: application/json" + -F "prompt=the subject turns toward the camera with smooth natural motion" + -F "num_frames=9" + -F "fps=24" + -F "num_inference_steps=2" + -F "guidance_scale=3.0" + -F "flow_shift=3.0" + -F "seed=42" +) +if [ -n "${INPUT_IMAGE:-}" ]; then + curl_args+=( + -F "input_reference=@${INPUT_IMAGE}" + -F 'extra_params={"size":"320x192"}' + ) +else + curl_args+=(-F "width=320" -F "height=192") +fi + +create_response="$(curl "${curl_args[@]}")" + +video_id="$(echo "${create_response}" | jq -er '.id')" +echo "Created video job ${video_id}" + +while true; do + status_response="$(curl -sS "${BASE_URL}/v1/videos/${video_id}")" + status="$(echo "${status_response}" | jq -er '.status')" + case "${status}" in + queued|in_progress) + sleep "${POLL_INTERVAL}" + ;; + completed) + break + ;; + failed) + echo "${status_response}" | jq . + exit 1 + ;; + *) + echo "Unexpected video job status: ${status}" + exit 1 + ;; + esac +done + +curl -sS -L "${BASE_URL}/v1/videos/${video_id}/content" -o "${OUTPUT_PATH}" +echo "Saved video to ${OUTPUT_PATH}" diff --git a/examples/online_serving/lingbot_video/run_curl_text_to_image.sh b/examples/online_serving/lingbot_video/run_curl_text_to_image.sh new file mode 100755 index 000000000..4c1543efa --- /dev/null +++ b/examples/online_serving/lingbot_video/run_curl_text_to_image.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8099}" +OUTPUT_PATH="${OUTPUT_PATH:-lingbot_t2i.png}" + +curl -sS -X POST "${BASE_URL}/v1/images/generations" \ + -H "Content-Type: application/json" \ + -d "{ + \"prompt\": \"a red fox standing in fresh snow\", + \"size\": \"320x192\", + \"num_inference_steps\": 2, + \"guidance_scale\": 3.0, + \"flow_shift\": 3.0, + \"seed\": 42, + \"response_format\": \"b64_json\" + }" | jq -er '.data[0].b64_json' | base64 -d > "${OUTPUT_PATH}" + +echo "Saved image to ${OUTPUT_PATH}" diff --git a/examples/online_serving/lingbot_video/run_server.sh b/examples/online_serving/lingbot_video/run_server.sh new file mode 100755 index 000000000..9bba0ef8d --- /dev/null +++ b/examples/online_serving/lingbot_video/run_server.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +MODEL="${MODEL:-robbyant/lingbot-video-dense-1.3b}" +PORT="${PORT:-8099}" + +vllm serve "${MODEL}" --omni \ + --model-class-name LingBotVideoPipeline \ + --port "${PORT}" diff --git a/tests/diffusion/models/lingbot_video/test_image_condition.py b/tests/diffusion/models/lingbot_video/test_image_condition.py new file mode 100644 index 000000000..0fbd11360 --- /dev/null +++ b/tests/diffusion/models/lingbot_video/test_image_condition.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from PIL import Image +from torch import nn + +from vllm_omni.diffusion.models.lingbot_video.image_condition import ( + apply_clean_prefix, + geometry_align_image, + prepare_ti2v_image_condition, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _gradient_image(width: int, height: int) -> Image.Image: + x = np.linspace(0, 255, width, dtype=np.uint8)[None, :] + y = np.linspace(0, 255, height, dtype=np.uint8)[:, None] + rgb = np.stack( + [ + np.broadcast_to(x, (height, width)), + np.broadcast_to(y, (height, width)), + np.full((height, width), 127, dtype=np.uint8), + ], + axis=-1, + ) + return Image.fromarray(rgb, mode="RGB") + + +@pytest.mark.parametrize("source_size", [(160, 64), (64, 160), (96, 96)]) +def test_geometry_align_image_supports_source_aspect_ratios(source_size): + pixel = geometry_align_image(_gradient_image(*source_size), 64, 96) + + assert pixel.shape == (1, 3, 1, 64, 96) + assert pixel.dtype == torch.float32 + assert 0 <= pixel.min() <= pixel.max() <= 1 + + +@pytest.mark.parametrize( + ("source_size", "target_size"), + [((160, 64), (64, 96)), ((64, 160), (96, 64))], +) +def test_geometry_align_image_center_crops_without_stretching(source_size, target_size): + width, height = source_size + target_height, target_width = target_size + rgb = np.full((height, width, 3), (255, 0, 0), dtype=np.uint8) + if width > target_width: + left = (width - target_width) // 2 + rgb[:, left : left + target_width] = (0, 255, 0) + else: + top = (height - target_height) // 2 + rgb[top : top + target_height] = (0, 255, 0) + + pixel = geometry_align_image( + Image.fromarray(rgb, mode="RGB"), + target_height, + target_width, + ) + + assert pixel.shape == (1, 3, 1, target_height, target_width) + expected_green = torch.tensor((0.0, 1.0, 0.0)).view(1, 3, 1, 1, 1) + assert torch.equal(pixel, expected_green.expand_as(pixel)) + + +class _LatentDistribution: + def __init__(self, latent: torch.Tensor): + self.latent = latent + + def sample(self, generator=None): + del generator + return self.latent + + +class _CapturingVAE(nn.Module): + def __init__(self): + super().__init__() + self.anchor = nn.Parameter(torch.zeros(())) + self.config = SimpleNamespace( + latents_mean=[0.0, 0.0, 0.0], + latents_std=[1.0, 1.0, 1.0], + ) + self.encoded_pixel = None + + def encode(self, pixel): + self.encoded_pixel = pixel.detach().clone() + return SimpleNamespace(latent_dist=_LatentDistribution(pixel)) + + +def test_ti2v_vlm_and_vae_paths_share_geometry_aligned_image(): + vae = _CapturingVAE() + condition = prepare_ti2v_image_condition( + _gradient_image(160, 64), + height=64, + width=96, + vae=vae, + vision_patch_size=16, + device=torch.device("cpu"), + ) + + vae_pixel = (vae.encoded_pixel + 1.0) / 2.0 + vae_image = vae_pixel[0, :, 0].permute(1, 2, 0).mul(255).byte().numpy() + assert condition.vlm_image.size == (96, 64) + np.testing.assert_allclose( + np.asarray(condition.vlm_image), + vae_image, + rtol=0, + atol=1, + ) + + +def test_apply_clean_prefix_only_overwrites_condition_frames(): + latents = torch.arange(20, dtype=torch.float32).reshape(1, 1, 5, 2, 2) + original_tail = latents[:, :, 2:].clone() + clean = torch.full((1, 1, 2, 2, 2), -3.0) + + returned = apply_clean_prefix(latents, clean) + + assert returned is latents + assert torch.equal(latents[:, :, :2], clean) + assert torch.equal(latents[:, :, 2:], original_tail) + + +def test_apply_clean_prefix_rejects_incompatible_shapes(): + with pytest.raises(ValueError, match="shape must match"): + apply_clean_prefix( + torch.zeros(1, 2, 3, 4, 4), + torch.zeros(1, 3, 1, 4, 4), + ) diff --git a/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py b/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py index 1322a542f..846a2642f 100644 --- a/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py +++ b/tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py @@ -6,9 +6,10 @@ from types import SimpleNamespace import numpy as np import pytest import torch +from PIL import Image from torch import nn -from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.request import DUMMY_DIFFUSION_REQUEST_ID, OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -22,6 +23,8 @@ def _make_pipeline(): nn.Module.__init__(pipeline) pipeline.device = torch.device("cpu") pipeline.default_negative_prompt = "default negative" + pipeline.default_image_negative_prompt = "default image negative" + pipeline.img_prompt_template = "" pipeline.od_config = SimpleNamespace(flow_shift=None) return pipeline @@ -37,17 +40,24 @@ def test_lingbot_video_pipeline_import_and_registry(): LingBotVideoPipeline, LingBotVideoTransformer3DModel, get_lingbot_video_post_process_func, + get_lingbot_video_pre_process_func, + ) + from vllm_omni.diffusion.registry import ( + _DIFFUSION_MODELS, + _DIFFUSION_POST_PROCESS_FUNCS, + _DIFFUSION_PRE_PROCESS_FUNCS, ) - from vllm_omni.diffusion.registry import _DIFFUSION_MODELS, _DIFFUSION_POST_PROCESS_FUNCS assert LingBotVideoPipeline is not None assert LingBotVideoTransformer3DModel is not None + assert get_lingbot_video_pre_process_func is not None assert get_lingbot_video_post_process_func is not None assert _DIFFUSION_MODELS["LingBotVideoPipeline"] == ( "lingbot_video", "pipeline_lingbot_video", "LingBotVideoPipeline", ) + assert _DIFFUSION_PRE_PROCESS_FUNCS["LingBotVideoPipeline"] == "get_lingbot_video_pre_process_func" assert _DIFFUSION_POST_PROCESS_FUNCS["LingBotVideoPipeline"] == "get_lingbot_video_post_process_func" @@ -58,6 +68,7 @@ def test_component_discovery_declarations(): assert LingBotVideoPipeline._encoder_modules == ["text_encoder"] assert LingBotVideoPipeline._vae_modules == ["vae"] assert LingBotVideoPipeline.supports_step_execution is False + assert LingBotVideoPipeline.max_outputs_per_prompt == 1 def test_extra_body_params_include_video_flow_shift_alias(): @@ -69,17 +80,47 @@ def test_extra_body_params_include_video_flow_shift_alias(): assert "batch_cfg" in params -def test_extract_prompt_string_and_mapping(): - from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import _extract_prompt +def test_preprocess_preserves_geometry_and_converts_rgb(tmp_path): + from vllm_omni.diffusion.models.lingbot_video import ( + get_lingbot_video_pre_process_func, + ) - string_req = SimpleNamespace(prompt="a robot waves") - assert _extract_prompt(string_req) == ("a robot waves", None) + image_path = tmp_path / "reference.png" + Image.new("L", (48, 32), color=127).save(image_path) + request = OmniDiffusionRequest( + prompt={ + "prompt": "a robot walks", + "modalities": ["video"], + "multi_modal_data": {"image": str(image_path)}, + }, + sampling_params=OmniDiffusionSamplingParams(), + request_id="lingbot-preprocess-test", + ) - mapping_req = SimpleNamespace(prompt={"prompt": "a robot walks", "negative_prompt": "low quality"}) - assert _extract_prompt(mapping_req) == ("a robot walks", "low quality") + result = get_lingbot_video_pre_process_func(SimpleNamespace())(request) - with pytest.raises(TypeError, match="Unsupported LingBot prompt type"): - _extract_prompt(SimpleNamespace(prompt=["not", "supported"])) + image = result.prompt["multi_modal_data"]["image"] + assert image.mode == "RGB" + assert image.size == (48, 32) + + +def test_preprocess_marks_internal_image_warmup_as_video(): + from vllm_omni.diffusion.models.lingbot_video import ( + get_lingbot_video_pre_process_func, + ) + + request = OmniDiffusionRequest( + prompt={ + "prompt": "dummy run", + "multi_modal_data": {"image": Image.new("RGB", (32, 32))}, + }, + sampling_params=OmniDiffusionSamplingParams(), + request_id=DUMMY_DIFFUSION_REQUEST_ID, + ) + + result = get_lingbot_video_pre_process_func(SimpleNamespace())(request) + + assert result.prompt["modalities"] == ["video"] def test_check_inputs_accepts_t2v_shapes_and_rejects_invalid_values(): @@ -104,6 +145,59 @@ def test_postprocess_keeps_torch_by_default_and_converts_np_when_requested(): array = post(frames, SimpleNamespace(output_type="np")) assert isinstance(array, np.ndarray) assert array.shape == (2, 4, 4, 3) + video = post({"video": frames}, SimpleNamespace(output_type="pt")) + assert set(video) == {"video"} + assert video["video"] is frames + image = post( + {"image": torch.ones(4, 4, 3)}, + SimpleNamespace(output_type="pt"), + ) + assert set(image) == {"image"} + assert isinstance(image["image"], np.ndarray) + assert image["image"].shape == (4, 4, 3) + latent = torch.ones(1, 4, 1, 2, 2) + latent_output = post({"image": latent}, SimpleNamespace(output_type="latent")) + assert set(latent_output) == {"image"} + assert latent_output["image"] is latent + + +def test_postprocess_preserves_image_contract_for_serving(): + from vllm_omni.diffusion.data import DiffusionOutput + from vllm_omni.diffusion.models.lingbot_video import get_lingbot_video_post_process_func + from vllm_omni.diffusion.output_formatter import ( + format_diffusion_outputs, + normalize_diffusion_postprocess_output, + ) + from vllm_omni.entrypoints.openai.api_server import _extract_images_from_result + + sampling = OmniDiffusionSamplingParams( + num_inference_steps=1, + output_type="pt", + resolution=64, + ) + request = OmniDiffusionRequest( + prompt={"prompt": "a robot", "modalities": ["image"]}, + sampling_params=sampling, + request_id="lingbot-image-contract-test", + ) + raw_output = {"image": torch.ones(8, 12, 3)} + processed = get_lingbot_video_post_process_func(SimpleNamespace())(raw_output, sampling) + normalized = normalize_diffusion_postprocess_output(processed) + + assert normalized.primary_key == "image" + [result] = format_diffusion_outputs( + request=request, + od_config=SimpleNamespace(model_class_name="LingBotVideoPipeline"), + diffusion_output=DiffusionOutput(output=raw_output), + output_data=raw_output, + postprocess_output=normalized, + ) + + assert len(result.images) == 1 + assert isinstance(result.images[0], np.ndarray) + endpoint_images = _extract_images_from_result(result) + assert len(endpoint_images) == 1 + assert endpoint_images[0].size == (12, 8) def test_forward_resolves_t2v_sampling_and_flow_shift_alias(): @@ -116,7 +210,11 @@ def test_forward_resolves_t2v_sampling_and_flow_shift_alias(): pipeline._generate = fake_generate req = _make_request_batch( - {"prompt": "a robot arm", "negative_prompt": "bad hands"}, + { + "prompt": "a robot arm", + "negative_prompt": "bad hands", + "modalities": ["video"], + }, height=192, width=320, num_frames=9, @@ -129,10 +227,12 @@ def test_forward_resolves_t2v_sampling_and_flow_shift_alias(): out = pipeline.forward(req) - assert torch.equal(out.output, torch.zeros(9, 192, 320, 3)) + assert torch.equal(out.output["video"], torch.zeros(9, 192, 320, 3)) assert len(calls) == 1 call = calls[0] assert call["prompt"] == "a robot arm" + assert call["mode"].value == "t2v" + assert call["input_image"] is None assert call["negative_prompt"] == "bad hands" assert call["height"] == 192 assert call["width"] == 320 @@ -194,6 +294,222 @@ def test_forward_prefers_shift_over_flow_shift_and_defaults_negative_prompt(): assert calls[0]["negative_prompt"] == "default negative" +@pytest.mark.parametrize( + ("prompt", "num_frames", "expected_mode", "output_key", "default_negative"), + [ + ( + {"prompt": "a still robot", "modalities": ["image"]}, + 1, + "t2i", + "image", + "default image negative", + ), + ( + { + "prompt": "a moving robot", + "modalities": ["video"], + "multi_modal_data": {"image": Image.new("RGB", (48, 32), color="blue")}, + }, + 9, + "ti2v", + "video", + "default negative", + ), + ], +) +def test_forward_dispatches_generation_mode_and_output( + prompt, + num_frames, + expected_mode, + output_key, + default_negative, +): + pipeline = _make_pipeline() + calls = [] + + def fake_generate(**kwargs): + calls.append(kwargs) + return torch.zeros(192, 320, 3) if output_key == "image" else torch.zeros(num_frames, 192, 320, 3) + + pipeline._generate = fake_generate + output = pipeline.forward( + _make_request_batch( + prompt, + height=192, + width=320, + num_frames=num_frames, + num_inference_steps=2, + guidance_scale=3.0, + seed=42, + output_type="pt", + ) + ) + + assert set(output.output) == {output_key} + assert calls[0]["mode"].value == expected_mode + assert calls[0]["negative_prompt"] == default_negative + if expected_mode == "ti2v": + assert isinstance(calls[0]["input_image"], Image.Image) + + +def test_batched_cfg_pads_image_conditioned_embeddings_and_masks(): + from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import ( + _batch_cfg_prompt_inputs, + ) + + positive = torch.ones(1, 7, 4) + positive_mask = torch.ones(1, 7, dtype=torch.long) + negative = torch.full((1, 9, 4), 2.0) + negative_mask = torch.ones(1, 9, dtype=torch.long) + + embeds, mask = _batch_cfg_prompt_inputs( + positive, + positive_mask, + negative, + negative_mask, + ) + + assert embeds.shape == (2, 9, 4) + assert mask.shape == (2, 9) + assert torch.equal(mask[0], torch.tensor([1, 1, 1, 1, 1, 1, 1, 0, 0])) + assert torch.equal(mask[1], torch.ones(9, dtype=torch.long)) + + +class _RecordingTransformer(nn.Module): + def __init__(self): + super().__init__() + self.anchor = nn.Parameter(torch.zeros(())) + self.config = SimpleNamespace(in_channels=1) + self.latent_inputs = [] + + def forward( + self, + latents, + timestep, + prompt_embeds, + *, + encoder_attention_mask, + return_dict, + ): + del timestep, prompt_embeds, encoder_attention_mask, return_dict + self.latent_inputs.append(latents.detach().clone()) + return (torch.zeros_like(latents),) + + +class _CorruptingScheduler: + sigma_max = 1.0 + sigma_min = 0.0 + + def set_timesteps(self, num_inference_steps, *, device, shift, **kwargs): + del num_inference_steps, shift, kwargs + self.timesteps = torch.tensor([1000.0, 500.0], device=device) + + def step( + self, + noise_pred, + timestep, + latents, + *, + return_dict, + generator, + ): + del noise_pred, timestep, return_dict, generator + return (latents + 7.0,) + + +@pytest.mark.parametrize( + ("guidance_scale", "batch_cfg", "expected_prompts", "expected_transformer_calls"), + [ + pytest.param(2.0, False, ["positive", "negative"], 4, id="sequential-cfg"), + pytest.param(2.0, True, ["positive", "negative"], 2, id="batched-cfg"), + pytest.param(1.0, False, ["positive"], 2, id="cfg-disabled"), + ], +) +def test_ti2v_reinjects_clean_prefix_across_cfg_modes( + guidance_scale, + batch_cfg, + expected_prompts, + expected_transformer_calls, +): + from vllm_omni.diffusion.models.lingbot_video import ( + LingBotGenerationMode, + LingBotImageCondition, + ) + + pipeline = _make_pipeline() + pipeline.transformer = _RecordingTransformer() + pipeline.scheduler = _CorruptingScheduler() + pipeline.progress_bar = lambda timesteps: timesteps + pipeline.prepare_latents = lambda *args, **kwargs: torch.zeros(1, 1, 3, 1, 1) + vlm_image = Image.new("RGB", (32, 32), color="red") + condition = LingBotImageCondition( + vlm_image=vlm_image, + clean_latent=torch.full((1, 1, 1, 1, 1), 5.0), + ) + pipeline.prepare_ti2v_image_condition = lambda *args, **kwargs: condition + encode_calls = [] + + def fake_encode(prompt, *, images, device): + del device + encode_calls.append((prompt, images)) + return torch.ones(1, 2, 4), torch.ones(1, 2, dtype=torch.long) + + pipeline.encode_prompt = fake_encode + + latents = pipeline._generate( + prompt="positive", + mode=LingBotGenerationMode.TI2V, + input_image=vlm_image, + negative_prompt="negative", + height=16, + width=16, + num_frames=9, + num_inference_steps=2, + guidance_scale=guidance_scale, + shift=3.0, + output_type="latent", + batch_cfg=batch_cfg, + ) + + assert [call[0] for call in encode_calls] == expected_prompts + assert all(call[1] == [vlm_image] for call in encode_calls) + assert len(pipeline.transformer.latent_inputs) == expected_transformer_calls + for item in pipeline.transformer.latent_inputs: + expected_prefix = condition.clean_latent.expand(item.shape[0], -1, -1, -1, -1) + assert torch.equal(item[:, :, :1], expected_prefix) + assert torch.equal(latents[:, :, :1], condition.clean_latent) + assert torch.equal(latents[:, :, 1:], torch.full((1, 1, 2, 1, 1), 14.0)) + + +def test_t2i_decodes_and_returns_the_unique_frame(): + from vllm_omni.diffusion.models.lingbot_video import LingBotGenerationMode + + pipeline = _make_pipeline() + pipeline.transformer = _RecordingTransformer() + pipeline.scheduler = _CorruptingScheduler() + pipeline.progress_bar = lambda timesteps: timesteps + pipeline.prepare_latents = lambda *args, **kwargs: torch.zeros(1, 1, 1, 2, 2) + pipeline.encode_prompt = lambda *args, **kwargs: ( + torch.ones(1, 2, 4), + torch.ones(1, 2, dtype=torch.long), + ) + pipeline._decode_latents = lambda latents: torch.ones(1, 16, 16, 3) + + image = pipeline._generate( + prompt="a still robot", + mode=LingBotGenerationMode.T2I, + height=16, + width=16, + num_frames=1, + num_inference_steps=2, + guidance_scale=1.0, + shift=3.0, + output_type="pt", + ) + + assert image.shape == (16, 16, 3) + + def test_forward_rejects_multi_request_batches(): pipeline = _make_pipeline() first = OmniDiffusionRequest( @@ -211,6 +527,45 @@ def test_forward_rejects_multi_request_batches(): pipeline.forward(DiffusionRequestBatch([first, second])) +def test_forward_marks_invalid_request_contract_as_client_error(): + from vllm_omni.errors import OmniClientError + + pipeline = _make_pipeline() + request = _make_request_batch( + { + "prompt": "unsupported image edit", + "modalities": ["image"], + "multi_modal_data": {"image": Image.new("RGB", (32, 32), color="blue")}, + }, + height=192, + width=320, + num_frames=1, + ) + + with pytest.raises(OmniClientError, match="does not accept") as exc_info: + pipeline.forward(request) + + assert exc_info.value.status_code == 400 + + +def test_forward_marks_unsupported_output_count_as_client_error(): + from vllm_omni.errors import OmniClientError + + pipeline = _make_pipeline() + request = _make_request_batch( + {"prompt": "motion", "modalities": ["video"]}, + height=192, + width=320, + num_frames=9, + num_outputs_per_prompt=2, + ) + + with pytest.raises(OmniClientError, match="one output per prompt") as exc_info: + pipeline.forward(request) + + assert exc_info.value.status_code == 400 + + def test_load_weights_rejects_external_weight_stream(): pipeline = _make_pipeline() diff --git a/tests/diffusion/models/lingbot_video/test_request_utils.py b/tests/diffusion/models/lingbot_video/test_request_utils.py new file mode 100644 index 000000000..cac769f05 --- /dev/null +++ b/tests/diffusion/models/lingbot_video/test_request_utils.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +from PIL import Image + +from vllm_omni.diffusion.models.lingbot_video.request_utils import ( + LingBotGenerationMode, + caption_from_lingbot_prompt, + normalize_lingbot_num_frames, + normalize_lingbot_request, + resolve_lingbot_mode, + resolve_lingbot_num_frames, + resolve_lingbot_output_dimensions, + resolve_lingbot_size, +) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _request(prompt, **sampling_overrides): + return SimpleNamespace( + prompt=prompt, + sampling_params=OmniDiffusionSamplingParams(**sampling_overrides), + ) + + +def _normalize(prompt, **sampling_overrides): + return normalize_lingbot_request( + _request(prompt, **sampling_overrides), + default_negative_prompt="video default", + default_image_negative_prompt="image default", + ) + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + ({"prompt": "still", "modalities": ["image"]}, LingBotGenerationMode.T2I), + ({"prompt": "motion", "modalities": ["video"]}, LingBotGenerationMode.T2V), + ( + { + "prompt": "motion", + "modalities": ["video"], + "multi_modal_data": {"image": Image.new("RGB", (8, 8))}, + }, + LingBotGenerationMode.TI2V, + ), + ], +) +def test_resolve_lingbot_mode(prompt, expected): + assert resolve_lingbot_mode(prompt) is expected + + +@pytest.mark.parametrize( + ("prompt", "message"), + [ + ( + {"prompt": "x", "modalities": ["image"], "multi_modal_data": {"image": Image.new("RGB", (8, 8))}}, + "does not accept", + ), + ( + { + "prompt": "x", + "modalities": ["video"], + "multi_modal_data": {"image": [Image.new("RGB", (8, 8)), Image.new("RGB", (8, 8))]}, + }, + "exactly one", + ), + ({"prompt": "x", "modalities": ["video"], "multi_modal_data": {"video": [1]}}, "reference modalities"), + ({"prompt": "x", "multi_modal_data": {"image": Image.new("RGB", (8, 8))}}, "explicitly set"), + ({"prompt": "x", "modalities": ["audio"]}, "output modality"), + ], +) +def test_resolve_lingbot_mode_rejects_invalid_media_combinations(prompt, message): + with pytest.raises(ValueError, match=message): + resolve_lingbot_mode(prompt) + + +def test_caption_from_lingbot_prompt_preserves_text_and_compact_json(): + assert caption_from_lingbot_prompt("plain text") == "plain text" + assert caption_from_lingbot_prompt({"caption": "caption only", "duration": 4}) == "caption only" + assert caption_from_lingbot_prompt({"caption": {"中文": True, "nested": {"value": 2}}}) == ( + '{"中文":true,"nested":{"value":2}}' + ) + assert ( + caption_from_lingbot_prompt( + {"scene": "海边", "camera": {"move": "pan"}, "duration": 5, "fps": 24, "size": "320x192"} + ) + == '{"scene":"海边","camera":{"move":"pan"}}' + ) + + +@pytest.mark.parametrize("prompt", [{"duration": 4, "fps": 24}, {"caption": ""}, ""]) +def test_caption_from_lingbot_prompt_rejects_empty_caption(prompt): + with pytest.raises(ValueError, match="empty|must not be empty"): + caption_from_lingbot_prompt(prompt) + + +@pytest.mark.parametrize( + ("num_frames", "expected"), + [(1, 1), (2, 5), (4, 5), (5, 5), (96, 97), (98, 101), (121, 121)], +) +def test_normalize_lingbot_num_frames_rounds_up_to_4n_plus_1(num_frames, expected): + assert normalize_lingbot_num_frames(num_frames) == expected + + +@pytest.mark.parametrize("num_frames", [0, -1, 1.5, "invalid"]) +def test_normalize_lingbot_num_frames_rejects_invalid_values(num_frames): + with pytest.raises(ValueError, match="num_frames"): + normalize_lingbot_num_frames(num_frames) + + +@pytest.mark.parametrize( + ("duration", "fps", "expected"), + [(4, 24, 97), (5, 24, 121), (8, 24, 193), (4, 12, 49), (4, 30, 121)], +) +def test_resolve_lingbot_num_frames_rounds_up(duration, fps, expected): + assert resolve_lingbot_num_frames(duration, fps) == expected + + +@pytest.mark.parametrize(("duration", "fps"), [(0, 24), (-1, 24), (0.01, 24), (1, 0), (1, -1)]) +def test_resolve_lingbot_num_frames_rejects_non_positive_or_empty_results(duration, fps): + with pytest.raises(ValueError, match="duration|fps|at least one"): + resolve_lingbot_num_frames(duration, fps) + + +@pytest.mark.parametrize( + ("resolution", "ratio", "expected"), + [ + ("192p", "1:1", (192, 192)), + ("192p", "9:16", (192, 320)), + ("192p", "16:9", (320, 192)), + ("192p", "3:4", (192, 256)), + ("192p", "4:3", (256, 192)), + ("480p", "1:1", (480, 480)), + ("480p", "9:16", (480, 832)), + ("480p", "16:9", (832, 480)), + ("480p", "3:4", (480, 640)), + ("480p", "4:3", (640, 480)), + ("720p", "1:1", (736, 736)), + ("720p", "9:16", (736, 1280)), + ("720p", "16:9", (1280, 736)), + ("720p", "3:4", (736, 960)), + ("720p", "4:3", (960, 736)), + ("1080p", "1:1", (1088, 1088)), + ("1080p", "9:16", (1088, 1920)), + ("1080p", "16:9", (1920, 1088)), + ("1080p", "3:4", (1088, 1440)), + ("1080p", "4:3", (1440, 1088)), + ("2k", "1:1", (1440, 1440)), + ("2k", "9:16", (1440, 2560)), + ("2k", "16:9", (2560, 1440)), + ("2k", "3:4", (1440, 1920)), + ("2k", "4:3", (1920, 1440)), + ("4k", "1:1", (2176, 2176)), + ("4k", "9:16", (2176, 3840)), + ("4k", "16:9", (3840, 2176)), + ("4k", "3:4", (2176, 2880)), + ("4k", "4:3", (2880, 2176)), + ], +) +def test_resolve_lingbot_size_supports_all_official_presets(resolution, ratio, expected): + assert resolve_lingbot_size(resolution=resolution, ratio=ratio) == expected + + +def test_resolve_lingbot_size_validates_sources_and_alignment(): + assert resolve_lingbot_size(size="320x192") == (192, 320) + assert resolve_lingbot_size(width=320, height=192) == (192, 320) + with pytest.raises(ValueError, match="provided together"): + resolve_lingbot_size(width=320) + with pytest.raises(ValueError, match="provided together"): + resolve_lingbot_size(resolution="192p") + with pytest.raises(ValueError, match="mutually exclusive"): + resolve_lingbot_size(width=320, height=192, resolution="192p", ratio="9:16") + with pytest.raises(ValueError, match="multiples of 16"): + resolve_lingbot_size(width=321, height=192) + + +def test_resolve_lingbot_output_dimensions_matches_request_priority(): + assert resolve_lingbot_output_dimensions( + sampling_width=480, + sampling_height=480, + prompt_fields={"resolution": "720p", "ratio": "16:9"}, + ) == (736, 1280) + + assert resolve_lingbot_output_dimensions( + sampling_width=480, + sampling_height=480, + prompt_fields={"width": 320, "height": 192}, + ) == (320, 192) + + assert resolve_lingbot_output_dimensions( + sampling_width=320, + sampling_height=192, + ) == (320, 192) + + assert resolve_lingbot_output_dimensions() == (480, 480) + + +def test_normalize_lingbot_request_uses_mode_defaults_and_preserves_empty_negative(): + t2i = _normalize( + {"prompt": "still", "modalities": ["image"], "negative_prompt": ""}, + width=320, + height=192, + num_frames=1, + ) + assert t2i.mode is LingBotGenerationMode.T2I + assert t2i.num_frames == 1 + assert t2i.negative_prompt == "" + + t2v = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + num_frames=9, + ) + assert t2v.mode is LingBotGenerationMode.T2V + assert t2v.negative_prompt == "video default" + + +def test_normalize_lingbot_request_rejects_explicit_zero_flow_shift(): + with pytest.raises(ValueError, match="shift.*positive"): + _normalize( + {"prompt": "still", "modalities": ["image"]}, + width=320, + height=192, + num_frames=1, + extra_args={"flow_shift": 0}, + ) + + +def test_normalize_lingbot_request_rejects_explicit_zero_width(): + with pytest.raises(ValueError, match="width"): + _normalize( + {"prompt": "still", "modalities": ["image"]}, + width=0, + height=192, + num_frames=1, + ) + + +def test_normalize_lingbot_request_rounds_all_video_frame_counts(): + config = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + num_frames=96, + fps=24, + frame_rate=24, + ) + assert config.num_frames == 97 + + config = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + num_frames=121, + fps=24, + frame_rate=24, + ) + assert config.num_frames == 121 + + +def test_normalize_lingbot_request_prefers_explicit_frames_over_duration(): + online = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + num_frames=9, + fps=24, + extra_args={"duration": 5}, + ) + assert online.num_frames == 9 + + offline = _normalize( + { + "prompt": {"caption": "motion", "duration": 5, "num_frames": 9}, + "modalities": ["video"], + }, + width=320, + height=192, + fps=24, + ) + assert offline.num_frames == 9 + + +def test_normalize_lingbot_request_uses_duration_then_model_default(): + duration = _normalize( + {"prompt": {"caption": "motion", "duration": 5}, "modalities": ["video"]}, + width=320, + height=192, + fps=24, + ) + assert duration.num_frames == 121 + + default = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + ) + assert default.num_frames == 81 + + +def test_normalize_lingbot_request_rejects_contract_conflicts(): + with pytest.raises(ValueError, match="text-to-image.*duration"): + _normalize( + {"prompt": {"caption": "still", "duration": 4}, "modalities": ["image"]}, + width=320, + height=192, + num_frames=1, + ) + with pytest.raises(ValueError, match="text-to-image requires"): + _normalize( + {"prompt": "still", "modalities": ["image"]}, + width=320, + height=192, + num_frames=5, + ) + with pytest.raises(ValueError, match="seconds.*duration"): + _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=320, + height=192, + num_frames=96, + extra_args={"seconds": 4, "duration": 4}, + ) + + +def test_normalize_lingbot_request_model_size_overrides_sampling_dimensions(): + # A prompt-provided size overrides the sampling width/height. + config = _normalize( + {"prompt": {"caption": "motion", "size": "320x192"}, "modalities": ["video"]}, + width=480, + height=480, + num_frames=81, + ) + assert (config.height, config.width) == (192, 320) + + # Without a prompt size, the sampling dimensions are used. + config = _normalize( + {"prompt": "motion", "modalities": ["video"]}, + width=480, + height=480, + num_frames=81, + ) + assert (config.height, config.width) == (480, 480) diff --git a/tests/e2e/online_serving/test_lingbot_video.py b/tests/e2e/online_serving/test_lingbot_video.py index e7af994cb..8f0f45639 100644 --- a/tests/e2e/online_serving/test_lingbot_video.py +++ b/tests/e2e/online_serving/test_lingbot_video.py @@ -1,18 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Online serving smoke for ``robbyant/lingbot-video-dense-1.3b``. - -The LingBot dense PR supports text-to-video only, so this module exercises the -native ``/v1/videos`` path with a small 9-frame request. -""" +"""Online serving smoke coverage for the dense LingBot-Video checkpoint.""" +import json import os import pytest +from tests.helpers.assertions import assert_video_first_frame_matches from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -43,11 +41,32 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.core_model -@pytest.mark.advanced_model +@pytest.mark.slow @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) -def test_text_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: +def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + openai_client.send_images_generations_http_request( + { + "json": { + "model": omni_server.model, + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "size": "320x192", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 2, + "guidance_scale": 3.0, + "flow_shift": 3.0, + "seed": 42, + } + } + ) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_video_generation_modes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: request_config = { "model": omni_server.model, "form_data": { @@ -65,3 +84,18 @@ def test_text_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientH }, } openai_client.send_video_diffusion_request(request_config) + + synthetic_image = generate_synthetic_image(320, 192, force_regenerate=True, seed=42) + request_config["form_data"]["prompt"] = "the red block moves slowly while the camera remains fixed" + # Keep TI2V dimensions model-scoped until /v1/videos forwards raw reference images. + request_config["form_data"].pop("height") + request_config["form_data"].pop("width") + request_config["form_data"]["extra_params"] = json.dumps({"size": "320x192"}, separators=(",", ":")) + request_config["image_reference"] = f"data:image/jpeg;base64,{synthetic_image['base64']}" + responses = openai_client.send_video_diffusion_request(request_config) + assert responses[0].videos + assert_video_first_frame_matches( + responses[0].videos[0], + synthetic_image["np_array"], + max_mean_absolute_error=0.25, + ) diff --git a/tests/e2e/online_serving/test_lingbot_video_expansion.py b/tests/e2e/online_serving/test_lingbot_video_expansion.py index cb5a314ff..fbf2bd5fa 100644 --- a/tests/e2e/online_serving/test_lingbot_video_expansion.py +++ b/tests/e2e/online_serving/test_lingbot_video_expansion.py @@ -4,9 +4,10 @@ """ L4 expansion coverage for ``robbyant/lingbot-video-dense-1.3b``. -This file remains dense-only. Basic single-GPU MoE serving is covered by -``test_lingbot_video_moe.py``; T2I, I2V, TI2V, and multi-GPU feature rows -belong in follow-up PRs. +This file remains dense-only. Basic T2I/T2V/TI2V serving for both dense and MoE +checkpoints is covered by ``test_lingbot_video.py`` and +``test_lingbot_video_moe.py``. Distributed feature rows belong in follow-up +PRs. """ import json diff --git a/tests/e2e/online_serving/test_lingbot_video_moe.py b/tests/e2e/online_serving/test_lingbot_video_moe.py index 32a6cbf2f..777880732 100644 --- a/tests/e2e/online_serving/test_lingbot_video_moe.py +++ b/tests/e2e/online_serving/test_lingbot_video_moe.py @@ -3,11 +3,14 @@ """Single-GPU serving smoke for ``robbyant/lingbot-video-moe-30b-a3b``.""" +import json import os import pytest +from tests.helpers.assertions import assert_video_first_frame_matches from tests.helpers.mark import hardware_marks +from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -38,10 +41,32 @@ def _get_server_cases(model: str): ] +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True) +def test_text_to_image_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: + openai_client.send_images_generations_http_request( + { + "json": { + "model": omni_server.model, + "prompt": PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "size": "320x192", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 2, + "guidance_scale": 3.0, + "flow_shift": 3.0, + "seed": 42, + } + } + ) + + @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_server_cases(MODEL), indirect=True) -def test_text_to_video_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: +def test_video_generation_modes_moe(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None: request_config = { "model": omni_server.model, "form_data": { @@ -57,3 +82,18 @@ def test_text_to_video_moe(omni_server: OmniServer, openai_client: OpenAIClientH }, } openai_client.send_video_diffusion_request(request_config) + + synthetic_image = generate_synthetic_image(320, 192, force_regenerate=True, seed=42) + request_config["form_data"]["prompt"] = "the red block moves slowly while the camera remains fixed" + # Keep TI2V dimensions model-scoped until /v1/videos forwards raw reference images. + request_config["form_data"].pop("height") + request_config["form_data"].pop("width") + request_config["form_data"]["extra_params"] = json.dumps({"size": "320x192"}, separators=(",", ":")) + request_config["image_reference"] = f"data:image/jpeg;base64,{synthetic_image['base64']}" + responses = openai_client.send_video_diffusion_request(request_config) + assert responses[0].videos + assert_video_first_frame_matches( + responses[0].videos[0], + synthetic_image["np_array"], + max_mean_absolute_error=0.25, + ) diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 2d6c0299e..19240c39c 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -223,6 +223,17 @@ def test_client(mock_async_diffusion): return TestClient(app) +@pytest.fixture +def lingbot_test_client(test_client): + test_client.app.state.stage_configs = [ + SimpleNamespace( + stage_type="diffusion", + engine_args={"model_class_name": "LingBotVideoPipeline"}, + ) + ] + return test_client + + @pytest.fixture def async_omni_test_client(): """Create test client with mocked AsyncOmni engine.""" diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index d16a51e00..3477cc16c 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -28,6 +28,7 @@ from vllm_omni.entrypoints.openai.api_server import router from vllm_omni.entrypoints.openai.protocol.videos import ( VideoGenerationRequest, VideoGenerationStatus, + VideoParams, VideoResponse, ) from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo @@ -440,6 +441,75 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, assert input_image.size == (96, 64) +def test_i2v_extra_params_dimensions_preserve_input_image_geometry(test_client, mocker: MockerFixture): + image_bytes = _make_test_image_bytes((48, 48)) + mocker.patch( + "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", + return_value=b"fake-video", + ) + response = test_client.post( + "/v1/videos", + data={ + "prompt": "A bear playing with yarn.", + "extra_params": json.dumps({"width": 96, "height": 64}), + }, + files={"input_reference": ("input.png", image_bytes, "image/png")}, + ) + + assert response.status_code == 200 + video_id = response.json()["id"] + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + + engine = test_client.app.state.openai_serving_video._engine_client + input_image = engine.captured_prompt["multi_modal_data"]["image"] + assert isinstance(input_image, Image.Image) + assert input_image.size == (48, 48) + sampling_params = engine.captured_sampling_params_list[0] + assert sampling_params.extra_args["width"] == 96 + assert sampling_params.extra_args["height"] == 64 + + +@pytest.mark.parametrize( + ("generation_request", "expected_num_frames", "expected_duration"), + [ + ( + VideoGenerationRequest(prompt="top-level frames", seconds="5", num_frames=9), + 9, + 5.0, + ), + ( + VideoGenerationRequest(prompt="nested frames", video_params=VideoParams(num_frames=9)), + 9, + None, + ), + ( + VideoGenerationRequest(prompt="seconds only", seconds="5"), + 120, + 5.0, + ), + ], +) +def test_video_generation_bridges_request_fields(generation_request, expected_num_frames, expected_duration): + engine = FakeAsyncOmni() + handler = OmniOpenAIServingVideo.for_diffusion( + diffusion_engine=engine, + model_name="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + ) + + asyncio.run(handler._run_and_extract(generation_request, "field-bridge")) + + sampling = engine.captured_sampling_params_list[0] + # Top-level ``seconds`` bridges into extra_args["duration"]; num_frames is + # passed through (or derived as seconds x fps when omitted). No private + # provenance channel is injected. + assert "_vllm_request_context" not in sampling.extra_args + assert sampling.num_frames == expected_num_frames + if expected_duration is None: + assert "duration" not in sampling.extra_args + else: + assert sampling.extra_args["duration"] == expected_duration + + def test_i2v_video_generation_with_image_reference_form(test_client, mocker: MockerFixture): mocker.patch( "vllm_omni.entrypoints.openai.serving_video._encode_video_bytes", diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index 9fe34f9b5..3b6fbe137 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: from tests.helpers.runtime import DiffusionResponse +import av import numpy as np import soundfile as sf from PIL import Image @@ -19,6 +20,7 @@ from PIL import Image from tests.helpers.media import ( convert_audio_bytes_to_text, cosine_similarity_text, + decode_b64_image, preprocess_text, ) @@ -112,6 +114,34 @@ def assert_image_diffusion_response( ) +def assert_images_generations_response( + response: dict[str, Any], + request_config: dict[str, Any], + run_level: str | None = None, +) -> None: + """Validate a successful ``/v1/images/generations`` JSON response.""" + del run_level + request_body = request_config.get("json") or {} + data = response.get("data") + assert isinstance(data, list), "Image generation response is missing data[]" + + expected_count = int(request_body.get("n", 1)) + assert len(data) == expected_count, f"Expected {expected_count} images, got {len(data)}" + + width = height = None + size = request_body.get("size") + if isinstance(size, str) and "x" in size: + width_value, height_value = size.lower().split("x", 1) + width, height = int(width_value), int(height_value) + + for item in data: + assert isinstance(item, dict), "Image generation data entries must be objects" + b64_json = item.get("b64_json") + assert isinstance(b64_json, str) and b64_json, "Image generation response is missing b64_json" + image = decode_b64_image(b64_json) + assert_image_valid(image, width=width, height=height) + + def assert_video_diffusion_response( response: "DiffusionResponse", request_config: dict[str, Any], @@ -158,6 +188,29 @@ def assert_video_diffusion_response( ) +def assert_video_first_frame_matches( + video: Path | bytes | BytesIO, + expected: np.ndarray, + *, + max_mean_absolute_error: float, +) -> None: + """Assert that an encoded video's first frame matches a reference image.""" + if isinstance(video, Path): + source: str | BytesIO = str(video) + else: + video_bytes = video if isinstance(video, bytes) else video.getvalue() + source = BytesIO(video_bytes) + + with av.open(source) as container: + first_frame = next(container.decode(video=0)).to_ndarray(format="rgb24") + + assert first_frame.shape == expected.shape + mean_absolute_error = float(np.abs(first_frame.astype(np.float32) - expected).mean() / 255.0) + assert mean_absolute_error < max_mean_absolute_error, ( + f"Expected first-frame MAE < {max_mean_absolute_error}, got {mean_absolute_error:.6f}." + ) + + def assert_audio_diffusion_response( response: "DiffusionResponse", request_config: dict[str, Any], diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index b01c152f5..a248e3952 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -41,6 +41,7 @@ from tests.helpers.assertions import ( assert_audio_speech_response, assert_diffusion_response, assert_http_error, + assert_images_generations_response, assert_omni_response, ) from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup @@ -1504,6 +1505,11 @@ class OpenAIClientHandler: err_code=cfg.get("err_code"), err_message=cfg.get("err_message"), ) + if cfg.get("err_code") is None: + assert resp.success, resp.error_message + payload = resp.json_body + assert isinstance(payload, dict) + assert_images_generations_response(payload, cfg, run_level=self.run_level) return [resp] def send_images_edits_http_request( diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index c73dbdf1e..503f02018 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -112,6 +112,90 @@ def test_sensenova_extra_registry_declares_request_and_response_params() -> None assert should_init_extra_args_for_non_diffusion_stages("SenseNovaU1Pipeline") is False +@pytest.mark.core_model +@pytest.mark.cpu +def test_lingbot_extra_registry_declares_request_params() -> None: + assert get_extra_body_params("LingBotVideoPipeline") == frozenset( + { + "batch_cfg", + "duration", + "flow_shift", + "negative_prompt", + "null_cond_clone_zero", + "offload_vae_during_denoise", + "output_type", + "refiner_sigma_tail_steps", + "resolution", + "ratio", + "shift", + "t_thresh", + } + ) + assert get_extra_output_params("LingBotVideoPipeline") == frozenset() + assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_lingbot_text_to_image_prompt_builder_preserves_empty_negative_prompt() -> None: + assert build_text_to_image_prompt( + "LingBotVideoPipeline", + prompt="a red fox in fresh snow", + negative_prompt="", + height=192, + width=320, + ) == { + "prompt": "a red fox in fresh snow", + "modalities": ["image"], + "negative_prompt": "", + } + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_lingbot_image_to_video_prompt_builder() -> None: + image = Image.new("RGB", (320, 192), "red") + result = build_image_to_video_prompt( + "LingBotVideoPipeline", + prompt="the fox looks toward the camera", + negative_prompt=None, + media_inputs={"image": image}, + height=192, + width=320, + num_frames=9, + ) + assert result == { + "prompt": "the fox looks toward the camera", + "modalities": ["video"], + "multi_modal_data": {"image": image}, + "num_frames": 9, + } + + +@pytest.mark.core_model +@pytest.mark.cpu +@pytest.mark.parametrize( + "media_inputs", + [ + {}, + {"image": "input.png"}, + {"image": [Image.new("RGB", (16, 16))]}, + {"image": Image.new("RGB", (16, 16)), "video": b"video"}, + ], + ids=["missing", "path", "multiple", "extra-modality"], +) +def test_lingbot_image_to_video_prompt_builder_rejects_invalid_media( + media_inputs: dict[str, object], +) -> None: + with pytest.raises(ValueError, match="exactly one PIL image"): + build_image_to_video_prompt( + "LingBotVideoPipeline", + prompt="move", + negative_prompt=None, + media_inputs=media_inputs, + ) + + @pytest.mark.core_model @pytest.mark.cpu @pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) diff --git a/vllm_omni/diffusion/models/lingbot_video/__init__.py b/vllm_omni/diffusion/models/lingbot_video/__init__.py index 013f7b2c3..0da570537 100644 --- a/vllm_omni/diffusion/models/lingbot_video/__init__.py +++ b/vllm_omni/diffusion/models/lingbot_video/__init__.py @@ -1,16 +1,46 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm_omni.diffusion.models.lingbot_video.image_condition import ( + LingBotImageCondition, + apply_clean_prefix, + geometry_align_image, + prepare_ti2v_image_condition, +) from vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer import ( LingBotVideoTransformer3DModel, ) from vllm_omni.diffusion.models.lingbot_video.pipeline_lingbot_video import ( LingBotVideoPipeline, get_lingbot_video_post_process_func, + get_lingbot_video_pre_process_func, +) +from vllm_omni.diffusion.models.lingbot_video.request_utils import ( + LingBotGenerationMode, + LingBotRequestConfig, + caption_from_lingbot_prompt, + normalize_lingbot_num_frames, + normalize_lingbot_request, + resolve_lingbot_num_frames, + resolve_lingbot_output_dimensions, + resolve_lingbot_size, ) __all__ = [ + "LingBotGenerationMode", + "LingBotImageCondition", + "LingBotRequestConfig", "LingBotVideoPipeline", "LingBotVideoTransformer3DModel", + "apply_clean_prefix", + "caption_from_lingbot_prompt", + "geometry_align_image", + "get_lingbot_video_pre_process_func", "get_lingbot_video_post_process_func", + "normalize_lingbot_num_frames", + "normalize_lingbot_request", + "prepare_ti2v_image_condition", + "resolve_lingbot_num_frames", + "resolve_lingbot_output_dimensions", + "resolve_lingbot_size", ] diff --git a/vllm_omni/diffusion/models/lingbot_video/image_condition.py b/vllm_omni/diffusion/models/lingbot_video/image_condition.py new file mode 100644 index 000000000..c99d00877 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_video/image_condition.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image + +IMAGE_MIN_TOKEN_NUM = 4 +IMAGE_MAX_TOKEN_NUM = 16384 +MAX_IMAGE_RATIO = 200 +SPATIAL_MERGE_SIZE = 2 + + +@dataclass(frozen=True) +class LingBotImageCondition: + vlm_image: Image.Image + clean_latent: torch.Tensor + + +def _round_by_factor(number: float, factor: int) -> int: + return round(number / factor) * factor + + +def _ceil_by_factor(number: float, factor: int) -> int: + return math.ceil(number / factor) * factor + + +def _floor_by_factor(number: float, factor: int) -> int: + return math.floor(number / factor) * factor + + +def smart_resize( + height: int, + width: int, + factor: int, + min_pixels: int | None = None, + max_pixels: int | None = None, +) -> tuple[int, int]: + max_pixels = max_pixels if max_pixels is not None else IMAGE_MAX_TOKEN_NUM * factor**2 + min_pixels = min_pixels if min_pixels is not None else IMAGE_MIN_TOKEN_NUM * factor**2 + if max_pixels < min_pixels: + raise ValueError("max_pixels must be greater than or equal to min_pixels.") + if min(height, width) <= 0: + raise ValueError(f"LingBot image height and width must be positive, got {height}x{width}.") + if max(height, width) / min(height, width) > MAX_IMAGE_RATIO: + raise ValueError(f"LingBot image absolute aspect ratio must be smaller than {MAX_IMAGE_RATIO}.") + + resized_height = max(factor, _round_by_factor(height, factor)) + resized_width = max(factor, _round_by_factor(width, factor)) + if resized_height * resized_width > max_pixels: + beta = math.sqrt((height * width) / max_pixels) + resized_height = _floor_by_factor(height / beta, factor) + resized_width = _floor_by_factor(width / beta, factor) + elif resized_height * resized_width < min_pixels: + beta = math.sqrt(min_pixels / (height * width)) + resized_height = _ceil_by_factor(height * beta, factor) + resized_width = _ceil_by_factor(width * beta, factor) + return resized_height, resized_width + + +def geometry_align_image( + image: Image.Image, + height: int, + width: int, +) -> torch.Tensor: + if not isinstance(image, Image.Image): + raise TypeError(f"LingBot input image must be a PIL image, got {type(image)!r}.") + if height <= 0 or width <= 0: + raise ValueError(f"LingBot target height and width must be positive, got {height}x{width}.") + + raw = torch.from_numpy(np.array(image.convert("RGB"))).permute(2, 0, 1).unsqueeze(0).contiguous() + old_height, old_width = raw.shape[-2:] + scale = max(height / old_height, width / old_width) + new_height = max(math.ceil(old_height * scale), height) + new_width = max(math.ceil(old_width * scale), width) + resized = F.interpolate( + raw, + size=(new_height, new_width), + mode="bilinear", + align_corners=False, + ) + top = int(round((new_height - height) / 2.0)) + left = int(round((new_width - width) / 2.0)) + cropped = resized[:, :, top : top + height, left : left + width] + return (cropped.float() / 255.0).unsqueeze(2) + + +def pixel_tensor_to_pil(pixel: torch.Tensor) -> Image.Image: + if pixel.ndim != 5 or pixel.shape[0] != 1 or pixel.shape[1] != 3: + raise ValueError(f"LingBot aligned image tensor must have shape [1,3,T,H,W], got {tuple(pixel.shape)}.") + frame = pixel[0, :, 0].detach().cpu().clamp(0, 1) + array = frame.permute(1, 2, 0).mul(255).byte().numpy() + return Image.fromarray(array, mode="RGB") + + +def vlm_image_from_aligned( + pixel: torch.Tensor, + *, + vision_patch_size: int, +) -> Image.Image: + image = pixel_tensor_to_pil(pixel) + patch_factor = int(vision_patch_size) * SPATIAL_MERGE_SIZE + if patch_factor <= 0: + raise ValueError(f"LingBot vision patch factor must be positive, got {patch_factor}.") + resized_height, resized_width = smart_resize( + image.height, + image.width, + factor=patch_factor, + ) + return image.resize((resized_width, resized_height)) + + +def _module_device(module: torch.nn.Module) -> torch.device: + try: + return next(module.parameters()).device + except StopIteration: + return torch.device("cpu") + + +@torch.no_grad() +def encode_clean_image_latent( + pixel: torch.Tensor, + *, + vae: torch.nn.Module, + generator: torch.Generator | None = None, +) -> torch.Tensor: + vae_device = _module_device(vae) + normalized = (pixel.to(device=vae_device, dtype=torch.float32) - 0.5) / 0.5 + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=vae_device.type == "cuda", + ): + encoded = vae.encode(normalized) + # Match the official LingBot TI2V path exactly: sample the posterior + # with the request generator before initial noise latents are created. + # Using mode() would change both the clean prefix and the RNG stream. + clean_latent = encoded.latent_dist.sample(generator) + + mean = torch.tensor( + vae.config.latents_mean, + device=clean_latent.device, + dtype=torch.float32, + ).view(1, -1, 1, 1, 1) + std_inv = ( + 1.0 + / torch.tensor( + vae.config.latents_std, + device=clean_latent.device, + dtype=torch.float32, + ) + ).view(1, -1, 1, 1, 1) + return (clean_latent.float() - mean) * std_inv + + +def prepare_ti2v_image_condition( + image: Image.Image, + *, + height: int, + width: int, + vae: torch.nn.Module, + vision_patch_size: int, + device: torch.device, + generator: torch.Generator | None = None, +) -> LingBotImageCondition: + pixel = geometry_align_image(image, height, width) + vlm_image = vlm_image_from_aligned( + pixel, + vision_patch_size=vision_patch_size, + ) + clean_latent = encode_clean_image_latent( + pixel, + vae=vae, + generator=generator, + ).to(device=device, dtype=torch.float32) + return LingBotImageCondition( + vlm_image=vlm_image, + clean_latent=clean_latent, + ) + + +def apply_clean_prefix( + latents: torch.Tensor, + clean_latent: torch.Tensor, +) -> torch.Tensor: + if latents.ndim != 5 or clean_latent.ndim != 5: + raise ValueError( + "LingBot latent and clean prefix must be 5D [B,C,T,H,W] tensors, " + f"got {tuple(latents.shape)} and {tuple(clean_latent.shape)}." + ) + if latents.shape[:2] != clean_latent.shape[:2] or latents.shape[3:] != clean_latent.shape[3:]: + raise ValueError( + "LingBot clean prefix batch/channel/spatial shape must match latents, " + f"got {tuple(clean_latent.shape)} for {tuple(latents.shape)}." + ) + prefix_length = int(clean_latent.shape[2]) + if prefix_length > latents.shape[2]: + raise ValueError(f"LingBot clean prefix length {prefix_length} exceeds latent length {latents.shape[2]}.") + latents[:, :, :prefix_length] = clean_latent.to( + device=latents.device, + dtype=latents.dtype, + ) + return latents diff --git a/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py b/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py index f54fa387c..1c94fecc6 100644 --- a/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py +++ b/vllm_omni/diffusion/models/lingbot_video/pipeline_lingbot_video.py @@ -14,17 +14,31 @@ import torch import torch.distributed as dist from diffusers import AutoencoderKLWan from diffusers.utils.torch_utils import randn_tensor +from PIL import Image from torch import nn from transformers import Qwen3VLForConditionalGeneration, Qwen3VLProcessor from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery +from vllm_omni.diffusion.models.interface import ( + SupportImageInput, + SupportsComponentDiscovery, +) +from vllm_omni.diffusion.models.lingbot_video.image_condition import ( + LingBotImageCondition, + apply_clean_prefix, + prepare_ti2v_image_condition, +) from vllm_omni.diffusion.models.lingbot_video.lingbot_video_transformer import LingBotVideoTransformer3DModel +from vllm_omni.diffusion.models.lingbot_video.request_utils import ( + LingBotGenerationMode, + normalize_lingbot_request, +) from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.errors import OmniClientError TOKEN_LENGTH = 37698 HIDDEN_STATE_SKIP_LAYER = 0 @@ -43,6 +57,7 @@ PROMPT_TEMPLATE = ( "commentary or evaluations:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" "<|im_start|>assistant\n" ) +IMG_PROMPT_TEMPLATE = "<|vision_start|><|image_pad|><|vision_end|>" DEFAULT_NEGATIVE_PROMPT = ( '{"universal_negative": {"visual_quality": ["low quality", "worst quality", "blurry", ' '"pixelated", "jpeg artifacts", "low resolution", "unstable color", "color flicker", ' @@ -57,6 +72,17 @@ DEFAULT_NEGATIVE_PROMPT = ( '"unrealistic texture", "deformed bottle", "liquid freezing improperly", ' '"distorted reflections"]}}' ) +DEFAULT_NEGATIVE_PROMPT_IMAGE = ( + '{"universal_negative": {"visual_quality": ["low quality", "worst quality", ' + '"blurry", "pixelated", "jpeg artifacts", "low resolution", "underexposed", ' + '"overexposed", "invisible subject", "subject hidden in darkness"], ' + '"artistic_style": ["painting", "illustration", "drawing", "cartoon", ' + '"3d render", "cgi", "sketch", "digital art"], "composition_and_content": ' + '["text", "watermark", "signature", "logo", "pillarboxed", "side bars", ' + '"portrait image in landscape frame"], "material_and_structure": ' + '["plastic-like glass", "unrealistic texture", "deformed bottle", ' + '"distorted reflections"]}}' +) def _dtype_from_name(value: Any, default: torch.dtype) -> torch.dtype: @@ -74,17 +100,6 @@ def _dtype_from_name(value: Any, default: torch.dtype) -> torch.dtype: raise ValueError(f"Unsupported LingBot dtype: {value!r}.") -def _extract_prompt(req: OmniDiffusionRequest) -> tuple[str, str | None]: - prompt_obj = req.prompt - if isinstance(prompt_obj, str): - return prompt_obj, None - if isinstance(prompt_obj, Mapping): - prompt = prompt_obj.get("prompt", "") - negative_prompt = prompt_obj.get("negative_prompt") - return str(prompt), None if negative_prompt is None else str(negative_prompt) - raise TypeError(f"Unsupported LingBot prompt type: {type(prompt_obj)!r}.") - - def _module_dtype(module: torch.nn.Module) -> torch.dtype: try: return next(module.parameters()).dtype @@ -213,19 +228,85 @@ def _batch_cfg_prompt_inputs( ) +def _load_lingbot_image(value: Any) -> Image.Image: + if isinstance(value, Image.Image): + return value.convert("RGB") + if isinstance(value, str): + try: + with Image.open(value) as image: + return image.convert("RGB") + except OSError as exc: + raise OmniClientError(f"Unable to load LingBot input image from {value!r}: {exc}.") from exc + raise OmniClientError(f"Unsupported LingBot image format {type(value)!r}; expected a PIL image or local path.") + + +def get_lingbot_video_pre_process_func(od_config: OmniDiffusionConfig): + del od_config + + def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest: + prompt = request.prompt + if isinstance(prompt, str): + return request + if not isinstance(prompt, Mapping): + raise OmniClientError(f"LingBot prompt must be a string or mapping, got {type(prompt)!r}.") + multi_modal_data = prompt.get("multi_modal_data") + if multi_modal_data is None: + return request + if not isinstance(multi_modal_data, Mapping): + raise OmniClientError("LingBot `multi_modal_data` must be a mapping.") + raw_images = multi_modal_data.get("image") + if raw_images is None: + return request + + multiple = isinstance(raw_images, (list, tuple)) + image_values = list(raw_images) if multiple else [raw_images] + images = [_load_lingbot_image(image) for image in image_values] + normalized_prompt = dict(prompt) + normalized_multi_modal_data = dict(multi_modal_data) + normalized_multi_modal_data["image"] = images if multiple else images[0] + normalized_prompt["multi_modal_data"] = normalized_multi_modal_data + if request.is_dummy_run() and "modalities" not in normalized_prompt: + normalized_prompt["modalities"] = ["video"] + request.prompt = normalized_prompt + return request + + return pre_process_func + + def get_lingbot_video_post_process_func(od_config: OmniDiffusionConfig): del od_config - def post_process_func(frames: torch.Tensor, sampling_params=None): + def post_process_func(frames: torch.Tensor | dict[str, torch.Tensor], sampling_params=None): + output_key = None + if isinstance(frames, dict): + output_keys = [key for key in ("image", "video") if key in frames] + if len(output_keys) != 1: + raise ValueError( + f"LingBot output must contain exactly one of 'image' or 'video', got {sorted(frames)!r}." + ) + output_key = output_keys[0] + frames = frames[output_key] output_type = getattr(sampling_params, "output_type", None) or "pt" - if output_type == "np" and isinstance(frames, torch.Tensor): - return frames.float().cpu().numpy() - return frames + # The image serving path currently accepts PIL images or NumPy arrays, + # while LingBot decodes T2I outputs to tensors. Keep that compatibility + # conversion without discarding the model's image/video payload key. + if ( + isinstance(frames, torch.Tensor) + and output_type != "latent" + and (output_type == "np" or output_key == "image") + ): + frames = frames.float().cpu().numpy() + return {output_key: frames} if output_key is not None else frames return post_process_func -class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscovery): +class LingBotVideoPipeline( + nn.Module, + SupportImageInput, + ProgressBarMixin, + SupportsComponentDiscovery, +): """Native vLLM-Omni entry for LingBot-Video checkpoints. The in-tree transformer supports both dense MLP blocks and routed MoE blocks. @@ -235,6 +316,7 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove """ supports_step_execution: ClassVar[bool] = False + max_outputs_per_prompt: ClassVar[int] = 1 _dit_modules: ClassVar[list[str]] = ["transformer"] _encoder_modules: ClassVar[list[str]] = ["text_encoder"] _vae_modules: ClassVar[list[str]] = ["vae"] @@ -249,6 +331,7 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove self.token_length = TOKEN_LENGTH self.hidden_state_skip_layer = HIDDEN_STATE_SKIP_LAYER self.prompt_template = PROMPT_TEMPLATE + self.img_prompt_template = IMG_PROMPT_TEMPLATE self._crop_start: int | None = None model = od_config.model @@ -298,6 +381,7 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove ) self.set_progress_bar_config(disable=bool(model_config.get("quiet_progress", True))) self.default_negative_prompt = DEFAULT_NEGATIVE_PROMPT + self.default_image_negative_prompt = DEFAULT_NEGATIVE_PROMPT_IMAGE def to(self, *args, **kwargs): device, dtype, non_blocking, _ = torch._C._nn._parse_to(*args, **kwargs) @@ -348,12 +432,18 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove self._crop_start = int(prefix["input_ids"].shape[1]) return self._crop_start - def _build_prompt_inputs(self, prompt: str | list[str]): + def _build_prompt_inputs( + self, + prompt: str | list[str], + *, + images: Any | None = None, + ): prompts = [prompt] if isinstance(prompt, str) else list(prompt) - texts = [self.apply_text_to_template(text, self.prompt_template) for text in prompts] + visual_template = self.img_prompt_template if images is not None else "" + texts = [self.apply_text_to_template(visual_template + text, self.prompt_template) for text in prompts] return self.processor( text=texts, - images=None, + images=images, videos=None, do_resize=False, truncation=True, @@ -367,10 +457,11 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove self, prompt: str | list[str], *, + images: Any | None = None, device: str | torch.device | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: device = torch.device(device) if device is not None else self.device - inputs = self._build_prompt_inputs(prompt).to(device) + inputs = self._build_prompt_inputs(prompt, images=images).to(device) outputs = self.text_encoder( **inputs, output_hidden_states=self.hidden_state_skip_layer is not None, @@ -392,6 +483,35 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove prompt_mask = prompt_mask[:, :true_len] return prompt_embeds, prompt_mask + def _vision_patch_size(self) -> int: + for obj in ( + getattr(getattr(self.text_encoder, "config", None), "vision_config", None), + getattr(getattr(self.processor, "image_processor", None), "config", None), + getattr(self.processor, "image_processor", None), + ): + patch_size = getattr(obj, "patch_size", None) + if patch_size is not None: + return int(patch_size) + return 16 + + def prepare_ti2v_image_condition( + self, + image: Image.Image, + *, + height: int, + width: int, + generator: torch.Generator | None = None, + ) -> LingBotImageCondition: + return prepare_ti2v_image_condition( + image, + height=height, + width=width, + vae=self.vae, + vision_patch_size=self._vision_patch_size(), + device=self.device, + generator=generator, + ) + def prepare_latents( self, num_frames: int, @@ -441,6 +561,8 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove self, *, prompt: str, + mode: LingBotGenerationMode, + input_image: Image.Image | None = None, negative_prompt: str = DEFAULT_NEGATIVE_PROMPT, height: int = 480, width: int = 480, @@ -467,6 +589,22 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove self.check_inputs(height, width, num_frames) device = self.device do_cfg = guidance_scale > 1.0 + image_condition = None + prompt_images = None + # Public image/mode combinations are validated by resolve_lingbot_mode. + # Keep these checks as defensive invariants for direct internal calls. + if mode is LingBotGenerationMode.TI2V: + if input_image is None: + raise ValueError("LingBot TI2V generation requires one input image.") + image_condition = self.prepare_ti2v_image_condition( + input_image, + height=height, + width=width, + generator=generator, + ) + prompt_images = [image_condition.vlm_image] + elif input_image is not None: + raise ValueError(f"LingBot {mode.value} generation does not accept an input image.") effective_batch_cfg = bool(batch_cfg) cfg_parallel = cfg_parallel_group is not None cfg_parallel_rank = 0 @@ -499,11 +637,19 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove if negative_prompt_embeds is not None: negative_embeds, negative_mask = negative_prompt_embeds, negative_prompt_mask else: - negative_embeds, negative_mask = self.encode_prompt(negative_prompt, device=device) + negative_embeds, negative_mask = self.encode_prompt( + negative_prompt, + images=prompt_images, + device=device, + ) prompt_embeds = prompt_mask = None else: if prompt_embeds is None: - prompt_embeds, prompt_mask = self.encode_prompt(prompt, device=device) + prompt_embeds, prompt_mask = self.encode_prompt( + prompt, + images=prompt_images, + device=device, + ) if do_cfg and not cfg_parallel: if null_cond_clone_zero: negative_embeds = torch.zeros_like(prompt_embeds) @@ -511,9 +657,15 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove elif negative_prompt_embeds is not None: negative_embeds, negative_mask = negative_prompt_embeds, negative_prompt_mask else: - negative_embeds, negative_mask = self.encode_prompt(negative_prompt, device=device) + negative_embeds, negative_mask = self.encode_prompt( + negative_prompt, + images=prompt_images, + device=device, + ) latents = self.prepare_latents(num_frames, height, width, generator, latents, device) + if image_condition is not None: + latents = apply_clean_prefix(latents, image_condition.clean_latent) sigmas = _compute_refiner_sigmas( sigma_max=float(self.scheduler.sigma_max), sigma_min=float(self.scheduler.sigma_min), @@ -620,7 +772,15 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove )[0].float() noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) - latents = self.scheduler.step(noise_pred, timestep, latents, return_dict=False, generator=generator)[0] + latents = self.scheduler.step( + noise_pred, + timestep, + latents, + return_dict=False, + generator=generator, + )[0] + if image_condition is not None: + latents = apply_clean_prefix(latents, image_condition.clean_latent) if cfg_parallel: dist.barrier(group=cfg_parallel_group) @@ -633,7 +793,14 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove if vae_offloaded and vae_restore_device is not None: self.vae.to(device=vae_restore_device) torch.accelerator.empty_cache() - return self._decode_latents(latents) + decoded = self._decode_latents(latents) + if mode is LingBotGenerationMode.T2I: + if decoded.shape[0] != 1: + raise RuntimeError( + f"LingBot T2I decode expected exactly one frame, got shape {tuple(decoded.shape)}." + ) + return decoded[0] + return decoded raise ValueError(f"Unsupported output_type: {output_type}") @torch.inference_mode() @@ -641,55 +808,67 @@ class LingBotVideoPipeline(nn.Module, ProgressBarMixin, SupportsComponentDiscove if req.num_reqs != 1: raise ValueError(f"LingBotVideoPipeline only supports one request per batch, got {req.num_reqs}.") request = req.requests[0] - prompt, prompt_negative = _extract_prompt(request) sampling = request.sampling_params extra_args = dict(sampling.extra_args or {}) + default_shift = getattr(self.od_config, "flow_shift", None) or 3.0 + default_output_type = getattr(self.od_config, "output_type", None) or "pt" + if default_output_type not in {"pt", "np", "latent"}: + default_output_type = "pt" + try: + if sampling.num_outputs_per_prompt != 1: + raise ValueError( + f"LingBotVideoPipeline only supports one output per prompt, got {sampling.num_outputs_per_prompt}." + ) + request_config = normalize_lingbot_request( + request, + default_negative_prompt=self.default_negative_prompt, + default_image_negative_prompt=self.default_image_negative_prompt, + default_shift=default_shift, + default_output_type=default_output_type, + ) + except (TypeError, ValueError) as exc: + raise OmniClientError(str(exc)) from exc + generator = sampling.generator if isinstance(generator, list): generator = generator[0] if generator else None if generator is None and sampling.seed is not None: generator = torch.Generator(device=self.device).manual_seed(int(sampling.seed)) - height = sampling.height if sampling.height is not None else extra_args.pop("height", 480) - width = sampling.width if sampling.width is not None else extra_args.pop("width", 480) - num_frames = sampling.num_frames or extra_args.pop("num_frames", 81) - num_inference_steps = ( - sampling.num_inference_steps - if sampling.num_inference_steps is not None - else extra_args.pop("num_inference_steps", 40) - ) - guidance_scale = ( - sampling.guidance_scale if sampling.guidance_scale_provided else extra_args.pop("guidance_scale", 6.0) - ) - shift = extra_args.pop( - "shift", - extra_args.pop("flow_shift", getattr(self.od_config, "flow_shift", None) or 3.0), - ) - negative_prompt = extra_args.pop("negative_prompt", prompt_negative or self.default_negative_prompt) - output_type = ( - sampling.output_type or getattr(self.od_config, "output_type", None) or extra_args.pop("output_type", "pt") - ) - if output_type not in {"pt", "np", "latent"}: - output_type = "pt" - sampling.output_type = output_type + sampling.height = request_config.height + sampling.width = request_config.width + sampling.num_frames = request_config.num_frames + sampling.fps = request_config.fps + sampling.frame_rate = float(request_config.fps) + sampling.num_inference_steps = request_config.num_inference_steps + sampling.guidance_scale = request_config.guidance_scale + sampling.output_type = request_config.output_type frames = self._generate( - prompt=prompt, - negative_prompt=negative_prompt, - height=height, - width=width, - num_frames=num_frames, - num_inference_steps=num_inference_steps, - guidance_scale=guidance_scale, - shift=shift, + prompt=request_config.prompt, + mode=request_config.mode, + input_image=request_config.input_image, + negative_prompt=request_config.negative_prompt, + height=request_config.height, + width=request_config.width, + num_frames=request_config.num_frames, + num_inference_steps=request_config.num_inference_steps, + guidance_scale=request_config.guidance_scale, + shift=request_config.shift, generator=generator, latents=sampling.latents, - output_type=output_type, - batch_cfg=bool(extra_args.pop("batch_cfg", False)), - null_cond_clone_zero=bool(extra_args.pop("null_cond_clone_zero", False)), - offload_vae_during_denoise=bool(extra_args.pop("offload_vae_during_denoise", False)), - t_thresh=extra_args.pop("t_thresh", None), - refiner_sigma_tail_steps=int(extra_args.pop("refiner_sigma_tail_steps", LOW_NOISE_TAIL_V1_DEFAULT_STEPS)), + output_type=request_config.output_type, + batch_cfg=bool(extra_args.get("batch_cfg", False)), + null_cond_clone_zero=bool(extra_args.get("null_cond_clone_zero", False)), + offload_vae_during_denoise=bool(extra_args.get("offload_vae_during_denoise", False)), + t_thresh=extra_args.get("t_thresh"), + refiner_sigma_tail_steps=int( + extra_args.get( + "refiner_sigma_tail_steps", + LOW_NOISE_TAIL_V1_DEFAULT_STEPS, + ) + ), ) - return DiffusionOutput(output=frames) + output_key = "image" if request_config.mode is LingBotGenerationMode.T2I else "video" + return DiffusionOutput(output={output_key: frames}) diff --git a/vllm_omni/diffusion/models/lingbot_video/request_utils.py b/vllm_omni/diffusion/models/lingbot_video/request_utils.py new file mode 100644 index 000000000..d522290f3 --- /dev/null +++ b/vllm_omni/diffusion/models/lingbot_video/request_utils.py @@ -0,0 +1,469 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from PIL import Image +from vllm.logger import init_logger + +logger = init_logger(__name__) + +LINGBOT_RUNTIME_PROMPT_FIELDS = frozenset( + { + "duration", + "seconds", + "fps", + "height", + "width", + "size", + "num_frames", + "resolution", + "ratio", + } +) + +# Values are stored as (height, width), matching the official LingBot runner. +LINGBOT_RESOLUTION_PRESETS: dict[str, dict[str, tuple[int, int]]] = { + "192p": { + "1:1": (192, 192), + "9:16": (192, 320), + "16:9": (320, 192), + "3:4": (192, 256), + "4:3": (256, 192), + }, + "480p": { + "1:1": (480, 480), + "9:16": (480, 832), + "16:9": (832, 480), + "3:4": (480, 640), + "4:3": (640, 480), + }, + "720p": { + "1:1": (736, 736), + "9:16": (736, 1280), + "16:9": (1280, 736), + "3:4": (736, 960), + "4:3": (960, 736), + }, + "1080p": { + "1:1": (1088, 1088), + "9:16": (1088, 1920), + "16:9": (1920, 1088), + "3:4": (1088, 1440), + "4:3": (1440, 1088), + }, + "2k": { + "1:1": (1440, 1440), + "9:16": (1440, 2560), + "16:9": (2560, 1440), + "3:4": (1440, 1920), + "4:3": (1920, 1440), + }, + "4k": { + "1:1": (2176, 2176), + "9:16": (2176, 3840), + "16:9": (3840, 2176), + "3:4": (2176, 2880), + "4:3": (2880, 2176), + }, +} + + +class LingBotGenerationMode(str, Enum): + T2I = "t2i" + T2V = "t2v" + TI2V = "ti2v" + + +@dataclass(frozen=True) +class LingBotRequestConfig: + mode: LingBotGenerationMode + prompt: str + negative_prompt: str + height: int + width: int + num_frames: int + fps: int + num_inference_steps: int + guidance_scale: float + shift: float + output_type: str + input_image: Image.Image | None + + +def _json_caption(value: Any) -> str: + try: + caption = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError) as exc: + raise ValueError(f"LingBot structured prompt is not JSON serializable: {value!r}.") from exc + if caption in {"", '""', "{}"}: + raise ValueError("LingBot structured prompt must contain a non-empty caption.") + return caption + + +def caption_from_lingbot_prompt(value: Any) -> str: + if isinstance(value, str): + if not value: + raise ValueError("LingBot `prompt` must not be empty.") + return value + if not isinstance(value, Mapping): + raise TypeError(f"LingBot `prompt` must be a string or mapping, got {type(value)!r}.") + + if "caption" in value: + caption = value["caption"] + if isinstance(caption, str): + if not caption: + raise ValueError("LingBot `caption` must not be empty.") + return caption + return _json_caption(caption) + + caption_fields = {key: item for key, item in value.items() if key not in LINGBOT_RUNTIME_PROMPT_FIELDS} + if not caption_fields: + raise ValueError("LingBot structured prompt is empty after removing runtime fields.") + return _json_caption(caption_fields) + + +def _prompt_envelope(prompt_obj: Any) -> Mapping[str, Any]: + return prompt_obj if isinstance(prompt_obj, Mapping) else {} + + +def _prompt_value(prompt_obj: Any) -> Any: + envelope = _prompt_envelope(prompt_obj) + return envelope["prompt"] if "prompt" in envelope else prompt_obj + + +def _image_items(envelope: Mapping[str, Any]) -> list[Any]: + multi_modal_data = envelope.get("multi_modal_data") or {} + if not isinstance(multi_modal_data, Mapping): + raise TypeError("LingBot `multi_modal_data` must be a mapping.") + images = multi_modal_data.get("image") + if images is None: + return [] + return list(images) if isinstance(images, (list, tuple)) else [images] + + +def resolve_lingbot_mode(prompt_obj: Any) -> LingBotGenerationMode: + envelope = _prompt_envelope(prompt_obj) + images = _image_items(envelope) + multi_modal_data = envelope.get("multi_modal_data") or {} + unsupported = [name for name in ("video", "audio") if multi_modal_data.get(name) is not None] + if unsupported: + raise ValueError(f"LingBot does not support reference modalities {unsupported!r}; only one image is allowed.") + + modalities = envelope.get("modalities") + if modalities is None: + if images: + raise ValueError("LingBot requests with an input image must explicitly set `modalities=['video']`.") + logger.warning_once( + "LingBot request omitted `modalities`; falling back to legacy text-to-video behavior. " + "Set `modalities=['video']` explicitly." + ) + return LingBotGenerationMode.T2V + if not isinstance(modalities, (list, tuple)) or len(modalities) != 1: + raise ValueError(f"LingBot `modalities` must contain exactly one output modality, got {modalities!r}.") + + modality = modalities[0] + if modality == "image": + if images: + raise ValueError(f"LingBot text-to-image does not accept reference images, got {len(images)}.") + return LingBotGenerationMode.T2I + if modality == "video": + if len(images) == 0: + return LingBotGenerationMode.T2V + if len(images) == 1: + return LingBotGenerationMode.TI2V + raise ValueError(f"LingBot text-image-to-video accepts exactly one reference image, got {len(images)}.") + raise ValueError(f"LingBot output modality must be 'image' or 'video', got {modality!r}.") + + +def _positive_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"LingBot `{name}` must be a positive integer, got {value!r}.") + try: + parsed = int(value) + numeric = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"LingBot `{name}` must be a positive integer, got {value!r}.") from exc + if not math.isfinite(numeric) or numeric != parsed or parsed <= 0: + raise ValueError(f"LingBot `{name}` must be a positive integer, got {value!r}.") + return parsed + + +def normalize_lingbot_num_frames(value: Any) -> int: + """Round a video length up to LingBot's causal VAE ``4n+1`` grid.""" + num_frames = _positive_int(value, "num_frames") + latent_frames = math.ceil((num_frames - 1) / 4) + return latent_frames * 4 + 1 + + +def resolve_lingbot_num_frames(duration: Any, fps: Any) -> int: + try: + duration_seconds = float(duration) + except (TypeError, ValueError) as exc: + raise ValueError(f"LingBot `duration` must be a positive number, got {duration!r}.") from exc + if not math.isfinite(duration_seconds) or duration_seconds <= 0: + raise ValueError(f"LingBot `duration` must be a positive number, got {duration!r}.") + fps_value = _positive_int(fps, "fps") + raw_frames = int(duration_seconds * fps_value) + if raw_frames < 1: + raise ValueError( + f"LingBot `duration` and `fps` must produce at least one frame, got duration={duration!r}, fps={fps!r}." + ) + return normalize_lingbot_num_frames(raw_frames) + + +def _parse_size(value: Any) -> tuple[int, int]: + if isinstance(value, str): + parts = value.lower().split("x") + if len(parts) != 2: + raise ValueError(f"LingBot `size` must use WIDTHxHEIGHT format, got {value!r}.") + width, height = parts + return _positive_int(height, "height"), _positive_int(width, "width") + if isinstance(value, (list, tuple)) and len(value) == 2: + width, height = value + return _positive_int(height, "height"), _positive_int(width, "width") + raise ValueError(f"LingBot `size` must use WIDTHxHEIGHT format, got {value!r}.") + + +def resolve_lingbot_size( + *, + width: Any = None, + height: Any = None, + size: Any = None, + resolution: Any = None, + ratio: Any = None, +) -> tuple[int, int]: + has_dimensions = width is not None or height is not None + has_size = size is not None + has_preset = resolution is not None or ratio is not None + if sum((has_dimensions, has_size, has_preset)) > 1: + raise ValueError("LingBot `size`, explicit `width`/`height`, and `resolution`/`ratio` are mutually exclusive.") + + if has_preset: + if resolution is None or ratio is None: + raise ValueError("LingBot `resolution` and `ratio` must be provided together.") + resolution_value = str(resolution).lower() + ratio_value = str(ratio) + if resolution_value not in LINGBOT_RESOLUTION_PRESETS: + raise ValueError( + f"Unsupported LingBot `resolution` {resolution!r}; choices are {sorted(LINGBOT_RESOLUTION_PRESETS)!r}." + ) + presets = LINGBOT_RESOLUTION_PRESETS[resolution_value] + if ratio_value not in presets: + raise ValueError( + f"Unsupported LingBot `ratio` {ratio!r} for resolution {resolution_value!r}; " + f"choices are {sorted(presets)!r}." + ) + resolved_height, resolved_width = presets[ratio_value] + elif has_size: + resolved_height, resolved_width = _parse_size(size) + else: + if width is None or height is None: + raise ValueError( + f"LingBot `width` and `height` must be provided together, got width={width!r}, height={height!r}." + ) + resolved_height = _positive_int(height, "height") + resolved_width = _positive_int(width, "width") + + if resolved_height % 16 != 0 or resolved_width % 16 != 0: + raise ValueError( + "LingBot `height` and `width` must be multiples of 16, " + f"got height={resolved_height}, width={resolved_width}." + ) + return resolved_height, resolved_width + + +def _runtime_prompt_fields(prompt_obj: Any) -> Mapping[str, Any]: + prompt_value = _prompt_value(prompt_obj) + return prompt_value if isinstance(prompt_value, Mapping) else {} + + +def _pick(mapping: Mapping[str, Any], key: str) -> Any: + value = mapping.get(key) + return value if value is not None else None + + +def _first_not_none(*values: Any) -> Any: + return next((value for value in values if value is not None), None) + + +def resolve_lingbot_output_dimensions( + *, + sampling_width: Any = None, + sampling_height: Any = None, + prompt_fields: Mapping[str, Any] | None = None, + default_width: int = 480, + default_height: int = 480, +) -> tuple[int, int]: + """Resolve the ``(width, height)`` that LingBot will use for generation.""" + prompt_fields = prompt_fields or {} + requested_width = _pick(prompt_fields, "width") + requested_height = _pick(prompt_fields, "height") + size = _pick(prompt_fields, "size") + resolution = _pick(prompt_fields, "resolution") + ratio = _pick(prompt_fields, "ratio") + if requested_width is not None or requested_height is not None: + width, height = requested_width, requested_height + elif size is not None or resolution is not None or ratio is not None: + width = height = None + else: + width, height = sampling_width, sampling_height + if width is None and height is None and size is None and resolution is None and ratio is None: + width, height = default_width, default_height + resolved_height, resolved_width = resolve_lingbot_size( + width=width, + height=height, + size=size, + resolution=resolution, + ratio=ratio, + ) + return resolved_width, resolved_height + + +def normalize_lingbot_request( + request: Any, + *, + default_negative_prompt: str, + default_image_negative_prompt: str, + default_height: int = 480, + default_width: int = 480, + default_num_frames: int = 81, + default_fps: int = 24, + default_num_inference_steps: int = 40, + default_guidance_scale: float = 6.0, + default_shift: float = 3.0, + default_output_type: str = "pt", +) -> LingBotRequestConfig: + prompt_obj = request.prompt + envelope = _prompt_envelope(prompt_obj) + prompt_fields = _runtime_prompt_fields(prompt_obj) + sampling = request.sampling_params + extra_args = dict(sampling.extra_args or {}) + + mode = resolve_lingbot_mode(prompt_obj) + prompt = caption_from_lingbot_prompt(_prompt_value(prompt_obj)) + images = _image_items(envelope) + input_image = images[0] if images else None + if input_image is not None and not isinstance(input_image, Image.Image): + raise TypeError(f"LingBot input image must be a PIL image after preprocessing, got {type(input_image)!r}.") + + prompt_negative = envelope.get("negative_prompt") + if "negative_prompt" in extra_args: + negative_prompt = extra_args["negative_prompt"] + elif prompt_negative is not None: + negative_prompt = prompt_negative + else: + negative_prompt = ( + default_image_negative_prompt if mode is LingBotGenerationMode.T2I else default_negative_prompt + ) + if not isinstance(negative_prompt, str): + raise TypeError(f"LingBot `negative_prompt` must be a string, got {type(negative_prompt)!r}.") + + fps_raw = _first_not_none( + _pick(extra_args, "fps"), + _pick(prompt_fields, "fps"), + sampling.resolved_frame_rate, + default_fps, + ) + fps = _positive_int(fps_raw, "fps") + + extra_seconds = _pick(extra_args, "seconds") + prompt_seconds = _pick(prompt_fields, "seconds") + duration_value = _pick(extra_args, "duration") + prompt_duration = _pick(prompt_fields, "duration") + if any(value is not None for value in (extra_seconds, prompt_seconds)) and any( + value is not None for value in (duration_value, prompt_duration) + ): + raise ValueError("LingBot `seconds` and `duration` cannot be provided together.") + duration = next( + (value for value in (extra_seconds, prompt_seconds, duration_value, prompt_duration) if value is not None), + None, + ) + + model_num_frames = _first_not_none( + _pick(extra_args, "num_frames"), + _pick(prompt_fields, "num_frames"), + ) + explicit_num_frames = model_num_frames + if explicit_num_frames is None and sampling.num_frames not in (None, 1): + # ``sampling.num_frames`` defaults to 1 (the image-model sentinel). + # Treat any other value as an explicit request; ``1`` means "omitted". + explicit_num_frames = sampling.num_frames + + if mode is LingBotGenerationMode.T2I: + if duration is not None: + raise ValueError(f"LingBot text-to-image does not accept `seconds` or `duration`, got {duration!r}.") + if explicit_num_frames is not None and _positive_int(explicit_num_frames, "num_frames") != 1: + raise ValueError(f"LingBot text-to-image requires `num_frames=1`, got {explicit_num_frames!r}.") + num_frames = 1 + elif explicit_num_frames is not None: + num_frames = normalize_lingbot_num_frames(explicit_num_frames) + elif duration is not None: + num_frames = resolve_lingbot_num_frames(duration, fps) + else: + num_frames = normalize_lingbot_num_frames(default_num_frames) + + width, height = resolve_lingbot_output_dimensions( + sampling_width=sampling.width, + sampling_height=sampling.height, + prompt_fields=prompt_fields, + default_width=default_width, + default_height=default_height, + ) + num_inference_steps = sampling.num_inference_steps + if num_inference_steps is None: + num_inference_steps = _first_not_none(_pick(extra_args, "num_inference_steps"), default_num_inference_steps) + num_inference_steps = _positive_int(num_inference_steps, "num_inference_steps") + guidance_scale = ( + sampling.guidance_scale + if sampling.guidance_scale_provided + else _first_not_none(_pick(extra_args, "guidance_scale"), default_guidance_scale) + ) + try: + guidance_scale = float(guidance_scale) + except (TypeError, ValueError) as exc: + raise ValueError(f"LingBot `guidance_scale` must be numeric, got {guidance_scale!r}.") from exc + if not math.isfinite(guidance_scale) or guidance_scale < 0: + raise ValueError(f"LingBot `guidance_scale` must be non-negative, got {guidance_scale!r}.") + + shift_raw = ( + extra_args["shift"] + if _pick(extra_args, "shift") is not None + else _first_not_none(_pick(extra_args, "flow_shift"), default_shift) + ) + try: + shift = float(shift_raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"LingBot `shift` must be numeric, got {shift_raw!r}.") from exc + if not math.isfinite(shift) or shift <= 0: + raise ValueError(f"LingBot `shift` must be positive, got {shift_raw!r}.") + + output_type = _first_not_none( + sampling.output_type, + _pick(extra_args, "output_type"), + default_output_type, + ) + if output_type not in {"pt", "np", "latent"}: + raise ValueError(f"LingBot `output_type` must be 'pt', 'np', or 'latent', got {output_type!r}.") + + return LingBotRequestConfig( + mode=mode, + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_frames=num_frames, + fps=fps, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + shift=shift, + output_type=output_type, + input_image=input_image, + ) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index a45f3bdee..3a6aeb7c2 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -586,6 +586,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = { "HeliosPipeline": "get_helios_pre_process_func", "HeliosPyramidPipeline": "get_helios_pre_process_func", "HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_pre_process_func", + "LingBotVideoPipeline": "get_lingbot_video_pre_process_func", "HunyuanImage3ForCausalMM": "get_hunyuan_image_3_pre_process_func", "MagiHumanPipeline": "get_magi_human_pre_process_func", "Cosmos3OmniDiffusersPipeline": "get_cosmos3_pre_process_func", diff --git a/vllm_omni/model_extras/lingbot_video.py b/vllm_omni/model_extras/lingbot_video.py index c7aae3763..b087e7073 100644 --- a/vllm_omni/model_extras/lingbot_video.py +++ b/vllm_omni/model_extras/lingbot_video.py @@ -1,16 +1,66 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from PIL import Image + LINGBOT_VIDEO_EXTRA_BODY_PARAMS = frozenset( { "batch_cfg", + "duration", "flow_shift", "negative_prompt", "null_cond_clone_zero", "offload_vae_during_denoise", "output_type", "refiner_sigma_tail_steps", + "resolution", + "ratio", "shift", "t_thresh", } ) + + +def build_text_to_image_prompt( + prompt: str, + negative_prompt: str | None, + height: int | None = None, + width: int | None = None, +) -> dict[str, Any]: + del height, width + result: dict[str, Any] = { + "prompt": prompt, + "modalities": ["image"], + } + if negative_prompt is not None: + result["negative_prompt"] = negative_prompt + return result + + +def build_image_to_video_prompt( + prompt: str, + negative_prompt: str | None, + media_inputs: Mapping[str, Any], + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, +) -> dict[str, Any]: + del height, width + if set(media_inputs) != {"image"} or not isinstance(media_inputs["image"], Image.Image): + raise ValueError("LingBot text-image-to-video requires exactly one PIL image input.") + + result: dict[str, Any] = { + "prompt": prompt, + "modalities": ["video"], + "multi_modal_data": {"image": media_inputs["image"]}, + } + if num_frames is not None: + result["num_frames"] = num_frames + if negative_prompt is not None: + result["negative_prompt"] = negative_prompt + return result diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index dab5b45cf..2e6db22e1 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -37,6 +37,12 @@ from vllm_omni.model_extras.helios import ( ) from vllm_omni.model_extras.hunyuan_image3 import build_x_to_text_prompt as build_hunyuan_x_to_text_prompt from vllm_omni.model_extras.lingbot_video import LINGBOT_VIDEO_EXTRA_BODY_PARAMS +from vllm_omni.model_extras.lingbot_video import ( + build_image_to_video_prompt as build_lingbot_image_to_video_prompt, +) +from vllm_omni.model_extras.lingbot_video import ( + build_text_to_image_prompt as build_lingbot_text_to_image_prompt, +) from vllm_omni.model_extras.ltx2 import LTX_EXTRA_BODY_PARAMS, LTX_EXTRA_OUTPUT_PARAMS from vllm_omni.model_extras.magi_human import ( MAGI_HUMAN_EXTRA_BODY_PARAMS, @@ -223,6 +229,8 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { }, "LingBotVideoPipeline": { "extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS, + "text_to_image_prompt_builder": build_lingbot_text_to_image_prompt, + "image_to_video_prompt_builder": build_lingbot_image_to_video_prompt, }, **{ model_class_name: {