From 280c4f56f61cb4c191e072ee9790bf11ce6b63bc Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Tue, 11 Aug 2026 17:39:40 +0800 Subject: [PATCH] [Misc] Build canonical image task prompts Signed-off-by: Hongsheng Liu --- .../image_to_video/image_to_video.py | 23 ++++- .../offline_inference/text_to_image/README.md | 5 +- .../text_to_image/text_to_image.py | 18 +++- .../test_image_task_prompts.py | 47 ++++++++++ tests/model_extras/test_model_extras.py | 76 +++++++++++----- vllm_omni/model_extras/__init__.py | 8 +- vllm_omni/model_extras/cosmos3.py | 26 ------ vllm_omni/model_extras/lingbot_video.py | 45 ---------- vllm_omni/model_extras/registry.py | 86 +++++++------------ 9 files changed, 177 insertions(+), 157 deletions(-) create mode 100644 tests/examples/offline_inference/test_image_task_prompts.py diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index b2f27bb5b..28e6b7447 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -61,7 +61,7 @@ from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.model_extras import ( - build_image_to_video_prompt, + adapt_image_to_video_prompt, get_extra_body_params, get_model_class_name, ) @@ -83,6 +83,22 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any] parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config") +def build_image_to_video_prompt( + prompt: str, + negative_prompt: str | None, + media_inputs: dict[str, Any], +) -> dict[str, Any]: + """Build the canonical request envelope for the shared I2V example.""" + result: dict[str, Any] = { + "prompt": prompt, + "modalities": ["video"], + "multi_modal_data": media_inputs, + } + if negative_prompt is not None: + result["negative_prompt"] = negative_prompt + return result + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -522,10 +538,13 @@ def main(): # Preserve the historical empty-prompt behavior for non-LTX examples. negative_prompt = "" prompt_dict = build_image_to_video_prompt( - model_class_name=model_class_name, prompt=args.prompt, negative_prompt=negative_prompt, media_inputs=media_inputs, + ) + prompt_dict = adapt_image_to_video_prompt( + model_class_name=model_class_name, + prompt=prompt_dict, height=height, width=width, num_frames=num_frames, diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index 4485e21c3..1a72b152c 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -57,8 +57,9 @@ python examples/offline_inference/text_to_image/text_to_image.py \ --output lingbot_t2i.png ``` -The LingBot prompt builder selects the `image` output modality and requests one -frame. Passing an input image is not supported in LingBot T2I mode. +The shared T2I entry point selects the `image` output modality. The LingBot +pipeline validates that T2I requests produce one frame and do not include an +input image. ## Quick Start diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 03bdf7080..fad89b085 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -21,7 +21,7 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.utils import stable_lora_int_id from vllm_omni.model_extras import ( - build_text_to_image_prompt, + adapt_text_to_image_prompt, get_extra_body_params, get_model_class_name, should_init_extra_args_for_non_diffusion_stages, @@ -56,6 +56,17 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any] parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config") +def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]: + """Build the canonical request envelope for the shared T2I example.""" + result: dict[str, Any] = { + "prompt": prompt, + "modalities": ["image"], + } + if negative_prompt is not None: + result["negative_prompt"] = negative_prompt + return result + + def _normalize_images_for_save(images: list[Any]) -> list[Any]: """Convert NumPy diffusion outputs to PIL images before saving.""" normalized = [] @@ -505,9 +516,12 @@ def main(): generation_start = time.perf_counter() prompt_dict = build_text_to_image_prompt( - model_class_name=model_class_name, prompt=args.prompt, negative_prompt=args.negative_prompt, + ) + prompt_dict = adapt_text_to_image_prompt( + model_class_name=model_class_name, + prompt=prompt_dict, height=args.height, width=args.width, ) diff --git a/tests/examples/offline_inference/test_image_task_prompts.py b/tests/examples/offline_inference/test_image_task_prompts.py new file mode 100644 index 000000000..57e8f1560 --- /dev/null +++ b/tests/examples/offline_inference/test_image_task_prompts.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import pytest +from PIL import Image + +from examples.offline_inference.image_to_video.image_to_video import build_image_to_video_prompt +from examples.offline_inference.text_to_image.text_to_image import build_text_to_image_prompt + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize( + ("negative_prompt", "expected_negative_prompt"), + [(None, None), ("", ""), ("blurry", "blurry")], +) +def test_text_to_image_builds_canonical_prompt( + negative_prompt: str | None, + expected_negative_prompt: str | None, +) -> None: + result = build_text_to_image_prompt("a red fox", negative_prompt) + + assert result["prompt"] == "a red fox" + assert result["modalities"] == ["image"] + if expected_negative_prompt is None: + assert "negative_prompt" not in result + else: + assert result["negative_prompt"] == expected_negative_prompt + + +def test_image_to_video_builds_canonical_prompt() -> None: + image = Image.new("RGB", (32, 16), "red") + + result = build_image_to_video_prompt( + prompt="the fox turns toward the camera", + negative_prompt="flicker", + media_inputs={"image": image}, + ) + + assert result == { + "prompt": "the fox turns toward the camera", + "modalities": ["video"], + "multi_modal_data": {"image": image}, + "negative_prompt": "flicker", + } diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 503f02018..6c8e0b6f8 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -3,15 +3,17 @@ from __future__ import annotations +from typing import Any + import pytest from PIL import Image from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.model_extras import ( + adapt_image_to_video_prompt, + adapt_text_to_image_prompt, build_image_to_image_prompt, - build_image_to_video_prompt, - build_text_to_image_prompt, build_x_to_text_prompt, get_extra_body_params, get_extra_output_params, @@ -20,6 +22,41 @@ from vllm_omni.model_extras import ( ) +def build_text_to_image_prompt( + model_class_name: str, + prompt: str, + negative_prompt: str | None, + height: int | None = None, + width: int | None = None, +) -> dict[str, Any]: + canonical_prompt: dict[str, Any] = { + "prompt": prompt, + "modalities": ["image"], + } + if negative_prompt is not None: + canonical_prompt["negative_prompt"] = negative_prompt + return adapt_text_to_image_prompt(model_class_name, canonical_prompt, height, width) + + +def build_image_to_video_prompt( + model_class_name: str, + prompt: str, + negative_prompt: str | None, + media_inputs: dict[str, Any], + height: int | None = None, + width: int | None = None, + num_frames: int | None = None, +) -> dict[str, Any]: + canonical_prompt: dict[str, Any] = { + "prompt": prompt, + "modalities": ["video"], + "multi_modal_data": media_inputs, + } + if negative_prompt is not None: + canonical_prompt["negative_prompt"] = negative_prompt + return adapt_image_to_video_prompt(model_class_name, canonical_prompt, height, width, num_frames) + + @pytest.mark.core_model @pytest.mark.cpu @pytest.mark.parametrize( @@ -137,7 +174,7 @@ def test_lingbot_extra_registry_declares_request_params() -> None: @pytest.mark.core_model @pytest.mark.cpu -def test_lingbot_text_to_image_prompt_builder_preserves_empty_negative_prompt() -> None: +def test_lingbot_text_to_image_uses_canonical_prompt() -> None: assert build_text_to_image_prompt( "LingBotVideoPipeline", prompt="a red fox in fresh snow", @@ -153,7 +190,7 @@ def test_lingbot_text_to_image_prompt_builder_preserves_empty_negative_prompt() @pytest.mark.core_model @pytest.mark.cpu -def test_lingbot_image_to_video_prompt_builder() -> None: +def test_lingbot_image_to_video_uses_canonical_prompt() -> None: image = Image.new("RGB", (320, 192), "red") result = build_image_to_video_prompt( "LingBotVideoPipeline", @@ -168,7 +205,6 @@ def test_lingbot_image_to_video_prompt_builder() -> None: "prompt": "the fox looks toward the camera", "modalities": ["video"], "multi_modal_data": {"image": image}, - "num_frames": 9, } @@ -184,16 +220,16 @@ def test_lingbot_image_to_video_prompt_builder() -> None: ], ids=["missing", "path", "multiple", "extra-modality"], ) -def test_lingbot_image_to_video_prompt_builder_rejects_invalid_media( - media_inputs: dict[str, object], +def test_lingbot_image_to_video_leaves_media_validation_to_the_pipeline( + media_inputs: dict[str, Any], ) -> None: - with pytest.raises(ValueError, match="exactly one PIL image"): - build_image_to_video_prompt( - "LingBotVideoPipeline", - prompt="move", - negative_prompt=None, - media_inputs=media_inputs, - ) + result = build_image_to_video_prompt( + "LingBotVideoPipeline", + prompt="move", + negative_prompt=None, + media_inputs=media_inputs, + ) + assert result["multi_modal_data"] is media_inputs @pytest.mark.core_model @@ -394,7 +430,7 @@ def test_ming_flash_omni_image_to_image_prompt_builder() -> None: @pytest.mark.core_model @pytest.mark.cpu @pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) -def test_cosmos3_text_to_image_prompt_builder_selects_image_modality(pipeline_name: str) -> None: +def test_cosmos3_text_to_image_uses_canonical_prompt(pipeline_name: str) -> None: assert build_text_to_image_prompt( pipeline_name, prompt="a red sports car at golden hour", @@ -543,14 +579,14 @@ def test_bagel_image_to_image_prompt_builder() -> None: @pytest.mark.core_model @pytest.mark.cpu -def test_unknown_pipeline_uses_default_text_to_image_prompt() -> None: +def test_unknown_pipeline_preserves_canonical_text_to_image_prompt() -> None: assert build_text_to_image_prompt( "UnknownPipeline", prompt="a cat", negative_prompt=None, height=512, width=512, - ) == {"prompt": "a cat"} + ) == {"prompt": "a cat", "modalities": ["image"]} @pytest.mark.core_model @@ -569,7 +605,7 @@ def test_unknown_pipeline_uses_default_image_to_image_prompt() -> None: } -def _build_vace_prompt(media_inputs: dict[str, object], *, num_frames: int = 5) -> dict: +def _build_vace_prompt(media_inputs: dict[str, Any], *, num_frames: int = 5) -> dict: return build_image_to_video_prompt( "WanVACEPipeline", prompt="a bird flying", @@ -597,7 +633,7 @@ def _build_vace_prompt(media_inputs: dict[str, object], *, num_frames: int = 5) ], ids=["i2v", "v2lf", "flf2v", "inpaint", "r2v"], ) -def test_vace_image_to_video_prompt_builder(media_inputs: dict[str, object]) -> None: +def test_vace_image_to_video_prompt_builder(media_inputs: dict[str, Any]) -> None: result = _build_vace_prompt(media_inputs) mmd = result["multi_modal_data"] if "reference_images" in mmd: @@ -616,7 +652,7 @@ def test_vace_image_to_video_prompt_builder(media_inputs: dict[str, object]) -> ({"control_image": Image.new("RGB", (320, 16))}, "Unsupported VACE media input"), ], ) -def test_vace_rejects_invalid_media_combinations(media_inputs: dict[str, object], message: str) -> None: +def test_vace_rejects_invalid_media_combinations(media_inputs: dict[str, Any], message: str) -> None: with pytest.raises(ValueError, match=message): _build_vace_prompt(media_inputs) diff --git a/vllm_omni/model_extras/__init__.py b/vllm_omni/model_extras/__init__.py index 1a3d71870..6ea807475 100644 --- a/vllm_omni/model_extras/__init__.py +++ b/vllm_omni/model_extras/__init__.py @@ -2,9 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from vllm_omni.model_extras.registry import ( + adapt_image_to_video_prompt, + adapt_text_to_image_prompt, build_image_to_image_prompt, - build_image_to_video_prompt, - build_text_to_image_prompt, build_x_to_text_prompt, get_extra_body_params, get_extra_output_params, @@ -14,9 +14,9 @@ from vllm_omni.model_extras.registry import ( ) __all__ = [ + "adapt_image_to_video_prompt", + "adapt_text_to_image_prompt", "build_image_to_image_prompt", - "build_image_to_video_prompt", - "build_text_to_image_prompt", "build_x_to_text_prompt", "get_extra_body_params", "get_extra_output_params", diff --git a/vllm_omni/model_extras/cosmos3.py b/vllm_omni/model_extras/cosmos3.py index a17e8193f..d0ffa5182 100644 --- a/vllm_omni/model_extras/cosmos3.py +++ b/vllm_omni/model_extras/cosmos3.py @@ -3,32 +3,6 @@ from __future__ import annotations -from typing import Any - - -def build_text_to_image_prompt( - prompt: str, - negative_prompt: str | None, - height: int | None = None, - width: int | None = None, -) -> dict[str, Any]: - """Build a Cosmos3 T2I prompt by selecting image output modality. - - Cosmos3 uses the same pipeline class for T2I and video modes. The OpenAI - image endpoint selects T2I by adding ``modalities=["image"]``. ``height`` - and ``width`` are accepted for registry compatibility; sizing travels - through sampling params instead of the prompt payload. - """ - del height, width - text_prompt: dict[str, Any] = { - "prompt": prompt, - "modalities": ["image"], - } - if negative_prompt is not None: - text_prompt["negative_prompt"] = negative_prompt - return text_prompt - - COSMOS3_EXTRA_BODY_PARAMS = frozenset( { "flow_shift", diff --git a/vllm_omni/model_extras/lingbot_video.py b/vllm_omni/model_extras/lingbot_video.py index b087e7073..c92f48bd2 100644 --- a/vllm_omni/model_extras/lingbot_video.py +++ b/vllm_omni/model_extras/lingbot_video.py @@ -3,11 +3,6 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import Any - -from PIL import Image - LINGBOT_VIDEO_EXTRA_BODY_PARAMS = frozenset( { "batch_cfg", @@ -24,43 +19,3 @@ LINGBOT_VIDEO_EXTRA_BODY_PARAMS = frozenset( "t_thresh", } ) - - -def build_text_to_image_prompt( - prompt: str, - negative_prompt: str | None, - height: int | None = None, - width: int | None = None, -) -> dict[str, Any]: - del height, width - result: dict[str, Any] = { - "prompt": prompt, - "modalities": ["image"], - } - if negative_prompt is not None: - result["negative_prompt"] = negative_prompt - return result - - -def build_image_to_video_prompt( - prompt: str, - negative_prompt: str | None, - media_inputs: Mapping[str, Any], - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, -) -> dict[str, Any]: - del height, width - if set(media_inputs) != {"image"} or not isinstance(media_inputs["image"], Image.Image): - raise ValueError("LingBot text-image-to-video requires exactly one PIL image input.") - - result: dict[str, Any] = { - "prompt": prompt, - "modalities": ["video"], - "multi_modal_data": {"image": media_inputs["image"]}, - } - if num_frames is not None: - result["num_frames"] = num_frames - if negative_prompt is not None: - result["negative_prompt"] = negative_prompt - return result diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 2e6db22e1..486550c9b 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -28,21 +28,12 @@ from vllm_omni.model_extras.cosmos3 import ( COSMOS3_EXTRA_BODY_PARAMS, COSMOS3_EXTRA_OUTPUT_PARAMS, ) -from vllm_omni.model_extras.cosmos3 import ( - build_text_to_image_prompt as build_cosmos3_text_to_image_prompt, -) from vllm_omni.model_extras.helios import ( HELIOS_EXTRA_BODY_PARAMS, HELIOS_EXTRA_OUTPUT_PARAMS, ) from vllm_omni.model_extras.hunyuan_image3 import build_x_to_text_prompt as build_hunyuan_x_to_text_prompt from vllm_omni.model_extras.lingbot_video import LINGBOT_VIDEO_EXTRA_BODY_PARAMS -from vllm_omni.model_extras.lingbot_video import ( - build_image_to_video_prompt as build_lingbot_image_to_video_prompt, -) -from vllm_omni.model_extras.lingbot_video import ( - build_text_to_image_prompt as build_lingbot_text_to_image_prompt, -) from vllm_omni.model_extras.ltx2 import LTX_EXTRA_BODY_PARAMS, LTX_EXTRA_OUTPUT_PARAMS from vllm_omni.model_extras.magi_human import ( MAGI_HUMAN_EXTRA_BODY_PARAMS, @@ -147,18 +138,6 @@ def build_x_to_text_prompt( return builder(model, prompt, has_image) -def default_text_to_image_prompt( - prompt: str, - negative_prompt: str | None, - height: int | None = None, - width: int | None = None, -) -> dict[str, Any]: - result: dict[str, Any] = {"prompt": prompt} - if negative_prompt is not None: - result["negative_prompt"] = negative_prompt - return result - - def default_image_to_image_prompt( prompt: str, negative_prompt: str | None, @@ -175,20 +154,6 @@ def default_image_to_image_prompt( return result -def default_image_to_video_prompt( - prompt: str, - negative_prompt: str | None, - media_inputs: Mapping[str, Any], - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, -) -> dict[str, Any]: - del height, width, num_frames - if set(media_inputs) != {"image"} or not isinstance(media_inputs["image"], Image.Image): - raise ValueError("This model only supports a single --image input in the shared image-to-video example.") - return default_image_to_image_prompt(prompt, negative_prompt, media_inputs["image"]) - - _EXTRA_SPECS: dict[str, dict[str, Any]] = { "AudioXPipeline": { "extra_body_params": AUDIOX_EXTRA_BODY_PARAMS, @@ -208,12 +173,11 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "Cosmos3OmniDiffusersPipeline": { "extra_body_params": COSMOS3_EXTRA_BODY_PARAMS, "extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS, - "text_to_image_prompt_builder": build_cosmos3_text_to_image_prompt, + # The shared T2I example already supplies modalities=["image"]. }, "Cosmos3OmniPipeline": { "extra_body_params": COSMOS3_EXTRA_BODY_PARAMS, "extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS, - "text_to_image_prompt_builder": build_cosmos3_text_to_image_prompt, }, "MagiHumanPipeline": { "extra_body_params": MAGI_HUMAN_EXTRA_BODY_PARAMS, @@ -229,8 +193,8 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { }, "LingBotVideoPipeline": { "extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS, - "text_to_image_prompt_builder": build_lingbot_text_to_image_prompt, - "image_to_video_prompt_builder": build_lingbot_image_to_video_prompt, + # Shared T2I/I2V envelopes select the output modality. LingBot's + # pipeline owns model-specific validation and normalization. }, **{ model_class_name: { @@ -303,20 +267,23 @@ def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False)) -def build_text_to_image_prompt( +def adapt_text_to_image_prompt( model_class_name: str | None, - prompt: str, - negative_prompt: str | None, + prompt: dict[str, Any], height: int | None = None, width: int | None = None, ) -> dict[str, Any]: + """Apply only model-specific changes to an example-owned T2I prompt.""" spec = _get_spec(model_class_name) - builder: TextToImagePromptBuilder = ( - spec.get("text_to_image_prompt_builder", default_text_to_image_prompt) - if spec is not None - else default_text_to_image_prompt + builder: TextToImagePromptBuilder | None = spec.get("text_to_image_prompt_builder") if spec else None + if builder is None: + return prompt + return builder( + prompt=str(prompt["prompt"]), + negative_prompt=prompt.get("negative_prompt"), + height=height, + width=width, ) - return builder(prompt, negative_prompt, height, width) def build_image_to_image_prompt( @@ -336,19 +303,26 @@ def build_image_to_image_prompt( return builder(prompt, negative_prompt, input_image, height, width) -def build_image_to_video_prompt( +def adapt_image_to_video_prompt( model_class_name: str | None, - prompt: str, - negative_prompt: str | None, - media_inputs: Mapping[str, Any], + prompt: dict[str, Any], height: int | None = None, width: int | None = None, num_frames: int | None = None, ) -> dict[str, Any]: + """Apply only model-specific changes to an example-owned I2V prompt.""" spec = _get_spec(model_class_name) - builder: ImageToVideoPromptBuilder = ( - spec.get("image_to_video_prompt_builder", default_image_to_video_prompt) - if spec is not None - else default_image_to_video_prompt + builder: ImageToVideoPromptBuilder | None = spec.get("image_to_video_prompt_builder") if spec else None + if builder is None: + return prompt + media_inputs = prompt.get("multi_modal_data") or {} + if not isinstance(media_inputs, Mapping): + raise TypeError("Canonical I2V prompt multi_modal_data must be a mapping.") + return builder( + prompt=str(prompt["prompt"]), + negative_prompt=prompt.get("negative_prompt"), + media_inputs=media_inputs, + height=height, + width=width, + num_frames=num_frames, ) - return builder(prompt, negative_prompt, media_inputs, height, width, num_frames)