diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index bcda12e2d..f876f2921 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -32,7 +32,7 @@ from vllm_omni.entrypoints.openai.protocol.videos import ( VideoParams, VideoResponse, ) -from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo +from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage from vllm_omni.entrypoints.openai.storage import LocalStorageManager from vllm_omni.entrypoints.openai.stores import AsyncDictStore, TaskRegistry from vllm_omni.errors import GuardrailViolationError @@ -433,6 +433,36 @@ def test_i2v_video_generation_form(test_client, mocker: MockerFixture): assert input_image.size == (48, 32) +def test_i2v_online_applies_model_specific_prompt_adapter() -> None: + engine = FakeAsyncOmni() + engine.model_class_name = "WanVACEPipeline" + handler = OmniOpenAIServingVideo.for_diffusion( + diffusion_engine=engine, + model_name="Wan-AI/Wan2.1-VACE-1.3B-diffusers", + ) + image = Image.new("RGB", (320, 16), "red") + + asyncio.run( + handler._run_and_extract( + VideoGenerationRequest( + prompt="A bird flying", + negative_prompt="blurry", + width=320, + height=16, + num_frames=5, + ), + "vace-online-adapter", + reference_image=ReferenceImage(image), + ) + ) + + prompt = engine.captured_prompt + assert prompt["negative_prompt"] == "blurry" + assert len(prompt["multi_modal_data"]["video"]) == 5 + assert len(prompt["multi_modal_data"]["mask"]) == 5 + assert prompt["multi_modal_data"]["video"][0] is image + + def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, mocker: MockerFixture): image_bytes = _make_test_image_bytes((48, 32)) diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index 6c8e0b6f8..3ca5ccbc2 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -22,41 +22,6 @@ from vllm_omni.model_extras import ( ) -def build_text_to_image_prompt( - model_class_name: str, - prompt: str, - negative_prompt: str | None, - height: int | None = None, - width: int | None = None, -) -> dict[str, Any]: - canonical_prompt: dict[str, Any] = { - "prompt": prompt, - "modalities": ["image"], - } - if negative_prompt is not None: - canonical_prompt["negative_prompt"] = negative_prompt - return adapt_text_to_image_prompt(model_class_name, canonical_prompt, height, width) - - -def build_image_to_video_prompt( - model_class_name: str, - prompt: str, - negative_prompt: str | None, - media_inputs: dict[str, Any], - height: int | None = None, - width: int | None = None, - num_frames: int | None = None, -) -> dict[str, Any]: - canonical_prompt: dict[str, Any] = { - "prompt": prompt, - "modalities": ["video"], - "multi_modal_data": media_inputs, - } - if negative_prompt is not None: - canonical_prompt["negative_prompt"] = negative_prompt - return adapt_image_to_video_prompt(model_class_name, canonical_prompt, height, width, num_frames) - - @pytest.mark.core_model @pytest.mark.cpu @pytest.mark.parametrize( @@ -172,66 +137,6 @@ def test_lingbot_extra_registry_declares_request_params() -> None: assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False -@pytest.mark.core_model -@pytest.mark.cpu -def test_lingbot_text_to_image_uses_canonical_prompt() -> None: - assert build_text_to_image_prompt( - "LingBotVideoPipeline", - prompt="a red fox in fresh snow", - negative_prompt="", - height=192, - width=320, - ) == { - "prompt": "a red fox in fresh snow", - "modalities": ["image"], - "negative_prompt": "", - } - - -@pytest.mark.core_model -@pytest.mark.cpu -def test_lingbot_image_to_video_uses_canonical_prompt() -> None: - image = Image.new("RGB", (320, 192), "red") - result = build_image_to_video_prompt( - "LingBotVideoPipeline", - prompt="the fox looks toward the camera", - negative_prompt=None, - media_inputs={"image": image}, - height=192, - width=320, - num_frames=9, - ) - assert result == { - "prompt": "the fox looks toward the camera", - "modalities": ["video"], - "multi_modal_data": {"image": image}, - } - - -@pytest.mark.core_model -@pytest.mark.cpu -@pytest.mark.parametrize( - "media_inputs", - [ - {}, - {"image": "input.png"}, - {"image": [Image.new("RGB", (16, 16))]}, - {"image": Image.new("RGB", (16, 16)), "video": b"video"}, - ], - ids=["missing", "path", "multiple", "extra-modality"], -) -def test_lingbot_image_to_video_leaves_media_validation_to_the_pipeline( - media_inputs: dict[str, Any], -) -> None: - result = build_image_to_video_prompt( - "LingBotVideoPipeline", - prompt="move", - negative_prompt=None, - media_inputs=media_inputs, - ) - assert result["multi_modal_data"] is media_inputs - - @pytest.mark.core_model @pytest.mark.cpu @pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) @@ -376,10 +281,13 @@ def test_ming_flash_omni_declared_extra_args_route_into_sampling_params() -> Non @pytest.mark.core_model @pytest.mark.cpu def test_ming_flash_omni_text_to_image_prompt_builder() -> None: - assert build_text_to_image_prompt( + assert adapt_text_to_image_prompt( "MingImagePipeline", - prompt="Please draw a cute cat.", - negative_prompt="ugly, blurry", + { + "prompt": "Please draw a cute cat.", + "modalities": ["image"], + "negative_prompt": "ugly, blurry", + }, height=512, width=768, ) == { @@ -394,10 +302,9 @@ def test_ming_flash_omni_text_to_image_prompt_builder() -> None: } # target_h/w are omitted when height/width are not supplied so the # pipeline's 1024x1024 default still applies. - assert build_text_to_image_prompt( + assert adapt_text_to_image_prompt( "MingImagePipeline", - prompt="Draw a poster.", - negative_prompt=None, + {"prompt": "Draw a poster.", "modalities": ["image"]}, ) == { "prompt": "Draw a poster.", "modalities": ["image"], @@ -427,28 +334,6 @@ def test_ming_flash_omni_image_to_image_prompt_builder() -> None: assert "negative_prompt" not in result -@pytest.mark.core_model -@pytest.mark.cpu -@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) -def test_cosmos3_text_to_image_uses_canonical_prompt(pipeline_name: str) -> None: - assert build_text_to_image_prompt( - pipeline_name, - prompt="a red sports car at golden hour", - negative_prompt="blurry, distorted", - height=1024, - width=1024, - ) == { - "prompt": "a red sports car at golden hour", - "modalities": ["image"], - "negative_prompt": "blurry, distorted", - } - assert build_text_to_image_prompt( - pipeline_name, - prompt="a red sports car", - negative_prompt=None, - ) == {"prompt": "a red sports car", "modalities": ["image"]} - - @pytest.mark.core_model @pytest.mark.cpu def test_audiox_extra_registry_declares_request_and_response_params() -> None: @@ -539,10 +424,13 @@ def test_unknown_pipeline_has_empty_extra_registry() -> None: @pytest.mark.core_model @pytest.mark.cpu def test_bagel_text_to_image_prompt_builder() -> None: - assert build_text_to_image_prompt( + assert adapt_text_to_image_prompt( "BagelPipeline", - prompt="a cat", - negative_prompt="blurry", + { + "prompt": "a cat", + "modalities": ["image"], + "negative_prompt": "blurry", + }, height=512, width=768, ) == { @@ -580,13 +468,16 @@ def test_bagel_image_to_image_prompt_builder() -> None: @pytest.mark.core_model @pytest.mark.cpu def test_unknown_pipeline_preserves_canonical_text_to_image_prompt() -> None: - assert build_text_to_image_prompt( - "UnknownPipeline", - prompt="a cat", - negative_prompt=None, - height=512, - width=512, - ) == {"prompt": "a cat", "modalities": ["image"]} + canonical_prompt = {"prompt": "a cat", "modalities": ["image"]} + assert ( + adapt_text_to_image_prompt( + "UnknownPipeline", + canonical_prompt, + height=512, + width=512, + ) + is canonical_prompt + ) @pytest.mark.core_model @@ -606,11 +497,13 @@ def test_unknown_pipeline_uses_default_image_to_image_prompt() -> None: def _build_vace_prompt(media_inputs: dict[str, Any], *, num_frames: int = 5) -> dict: - return build_image_to_video_prompt( + return adapt_image_to_video_prompt( "WanVACEPipeline", - prompt="a bird flying", - negative_prompt=None, - media_inputs=media_inputs, + { + "prompt": "a bird flying", + "modalities": ["video"], + "multi_modal_data": media_inputs, + }, height=16, width=320, num_frames=num_frames, @@ -698,10 +591,9 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N assert get_extra_output_params(model_class_name) == frozenset() assert should_init_extra_args_for_non_diffusion_stages(model_class_name) is True - wrapper_prompt = build_text_to_image_prompt( + wrapper_prompt = adapt_text_to_image_prompt( "MammothModa2ForConditionalGeneration", - prompt="a cat", - negative_prompt=None, + {"prompt": "a cat", "modalities": ["image"]}, height=256, width=256, ) @@ -713,10 +605,13 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N def test_mammothmoda2_text_to_image_prompt_builder() -> None: # Image dims are converted to the AR grid (width/16 x height/16); the negative # prompt is ignored (MammothModa2 t2i uses CFG, not an explicit negative path). - assert build_text_to_image_prompt( + assert adapt_text_to_image_prompt( "MammothModa2DiTPipeline", - prompt="a cat", - negative_prompt="blurry", + { + "prompt": "a cat", + "modalities": ["image"], + "negative_prompt": "blurry", + }, height=512, width=768, ) == { diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index 3a25283c8..a86205360 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -30,6 +30,7 @@ from vllm_omni.entrypoints.openai.stage_params import ( from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request from vllm_omni.entrypoints.openai.video_api_utils import _encode_video_bytes, encode_video_base64 from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt +from vllm_omni.model_extras import adapt_image_to_video_prompt from vllm_omni.outputs.output_metadata import ( DiffusionMetadataMapping, DiffusionMultimodalOutput, @@ -100,11 +101,14 @@ class OmniOpenAIServingVideo: if self._stage_configs is None and stage_configs is not None: self._stage_configs = stage_configs + def _get_diffusion_od_config(self) -> object | None: + get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) + return get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None) + @property def supports_mixed_reference_inputs(self) -> bool: """Return whether the configured diffusion model accepts mixed refs.""" - get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None) - od_config = get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None) + od_config = self._get_diffusion_od_config() if od_config is None: return False @@ -252,6 +256,18 @@ class OmniOpenAIServingVideo: loggable = {**loggable, **redacted} logger.info("Applied extra_params: %s", loggable) + model_class_name = getattr(self._get_diffusion_od_config(), "model_class_name", None) + prompt = cast( + OmniTextPrompt, + adapt_image_to_video_prompt( + model_class_name=model_class_name, + prompt=prompt, + height=gen_params.height, + width=gen_params.width, + num_frames=gen_params.num_frames, + ), + ) + self._apply_lora(request.lora, gen_params) logger.info(