anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Misc] Address prompt adapter review feedback

Browse Source 
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu committed 2026-08-11 23:41:44 +08:00
1 parent 280c4f56f6
commit a4516ea5db
3 files changed
+87 -146
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/entrypoints/openai_api/test_video_server.py
+31 -1
View File
@@ -32,7 +32,7 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
VideoParams,
VideoResponse,
)
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage
from vllm_omni.entrypoints.openai.storage import LocalStorageManager
from vllm_omni.entrypoints.openai.stores import AsyncDictStore, TaskRegistry
from vllm_omni.errors import GuardrailViolationError
@@ -433,6 +433,36 @@ def test_i2v_video_generation_form(test_client, mocker: MockerFixture):
assert input_image.size == (48, 32)
def test_i2v_online_applies_model_specific_prompt_adapter() -> None:
engine = FakeAsyncOmni()
engine.model_class_name = "WanVACEPipeline"
handler = OmniOpenAIServingVideo.for_diffusion(
diffusion_engine=engine,
model_name="Wan-AI/Wan2.1-VACE-1.3B-diffusers",
)
image = Image.new("RGB", (320, 16), "red")
asyncio.run(
handler._run_and_extract(
VideoGenerationRequest(
prompt="A bird flying",
negative_prompt="blurry",
width=320,
height=16,
num_frames=5,
),
"vace-online-adapter",
reference_image=ReferenceImage(image),
)
)
prompt = engine.captured_prompt
assert prompt["negative_prompt"] == "blurry"
assert len(prompt["multi_modal_data"]["video"]) == 5
assert len(prompt["multi_modal_data"]["mask"]) == 5
assert prompt["multi_modal_data"]["video"][0] is image
def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client, mocker: MockerFixture):
image_bytes = _make_test_image_bytes((48, 32))
tests/model_extras/test_model_extras.py
+38 -143
View File
@@ -22,41 +22,6 @@ from vllm_omni.model_extras import (
)
def build_text_to_image_prompt(
model_class_name: str,
prompt: str,
negative_prompt: str | None,
height: int | None = None,
width: int | None = None,
) -> dict[str, Any]:
canonical_prompt: dict[str, Any] = {
"prompt": prompt,
"modalities": ["image"],
}
if negative_prompt is not None:
canonical_prompt["negative_prompt"] = negative_prompt
return adapt_text_to_image_prompt(model_class_name, canonical_prompt, height, width)
def build_image_to_video_prompt(
model_class_name: str,
prompt: str,
negative_prompt: str | None,
media_inputs: dict[str, Any],
height: int | None = None,
width: int | None = None,
num_frames: int | None = None,
) -> dict[str, Any]:
canonical_prompt: dict[str, Any] = {
"prompt": prompt,
"modalities": ["video"],
"multi_modal_data": media_inputs,
}
if negative_prompt is not None:
canonical_prompt["negative_prompt"] = negative_prompt
return adapt_image_to_video_prompt(model_class_name, canonical_prompt, height, width, num_frames)
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize(
@@ -172,66 +137,6 @@ def test_lingbot_extra_registry_declares_request_params() -> None:
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
@pytest.mark.core_model
@pytest.mark.cpu
def test_lingbot_text_to_image_uses_canonical_prompt() -> None:
assert build_text_to_image_prompt(
"LingBotVideoPipeline",
prompt="a red fox in fresh snow",
negative_prompt="",
height=192,
width=320,
) == {
"prompt": "a red fox in fresh snow",
"modalities": ["image"],
"negative_prompt": "",
}
@pytest.mark.core_model
@pytest.mark.cpu
def test_lingbot_image_to_video_uses_canonical_prompt() -> None:
image = Image.new("RGB", (320, 192), "red")
result = build_image_to_video_prompt(
"LingBotVideoPipeline",
prompt="the fox looks toward the camera",
negative_prompt=None,
media_inputs={"image": image},
height=192,
width=320,
num_frames=9,
)
assert result == {
"prompt": "the fox looks toward the camera",
"modalities": ["video"],
"multi_modal_data": {"image": image},
}
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize(
"media_inputs",
[
{},
{"image": "input.png"},
{"image": [Image.new("RGB", (16, 16))]},
{"image": Image.new("RGB", (16, 16)), "video": b"video"},
],
ids=["missing", "path", "multiple", "extra-modality"],
)
def test_lingbot_image_to_video_leaves_media_validation_to_the_pipeline(
media_inputs: dict[str, Any],
) -> None:
result = build_image_to_video_prompt(
"LingBotVideoPipeline",
prompt="move",
negative_prompt=None,
media_inputs=media_inputs,
)
assert result["multi_modal_data"] is media_inputs
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
@@ -376,10 +281,13 @@ def test_ming_flash_omni_declared_extra_args_route_into_sampling_params() -> Non
@pytest.mark.core_model
@pytest.mark.cpu
def test_ming_flash_omni_text_to_image_prompt_builder() -> None:
assert build_text_to_image_prompt(
assert adapt_text_to_image_prompt(
"MingImagePipeline",
prompt="Please draw a cute cat.",
negative_prompt="ugly, blurry",
{
"prompt": "Please draw a cute cat.",
"modalities": ["image"],
"negative_prompt": "ugly, blurry",
},
height=512,
width=768,
) == {
@@ -394,10 +302,9 @@ def test_ming_flash_omni_text_to_image_prompt_builder() -> None:
}
# target_h/w are omitted when height/width are not supplied so the
# pipeline's 1024x1024 default still applies.
assert build_text_to_image_prompt(
assert adapt_text_to_image_prompt(
"MingImagePipeline",
prompt="Draw a poster.",
negative_prompt=None,
{"prompt": "Draw a poster.", "modalities": ["image"]},
) == {
"prompt": "Draw a poster.",
"modalities": ["image"],
@@ -427,28 +334,6 @@ def test_ming_flash_omni_image_to_image_prompt_builder() -> None:
assert "negative_prompt" not in result
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
def test_cosmos3_text_to_image_uses_canonical_prompt(pipeline_name: str) -> None:
assert build_text_to_image_prompt(
pipeline_name,
prompt="a red sports car at golden hour",
negative_prompt="blurry, distorted",
height=1024,
width=1024,
) == {
"prompt": "a red sports car at golden hour",
"modalities": ["image"],
"negative_prompt": "blurry, distorted",
}
assert build_text_to_image_prompt(
pipeline_name,
prompt="a red sports car",
negative_prompt=None,
) == {"prompt": "a red sports car", "modalities": ["image"]}
@pytest.mark.core_model
@pytest.mark.cpu
def test_audiox_extra_registry_declares_request_and_response_params() -> None:
@@ -539,10 +424,13 @@ def test_unknown_pipeline_has_empty_extra_registry() -> None:
@pytest.mark.core_model
@pytest.mark.cpu
def test_bagel_text_to_image_prompt_builder() -> None:
assert build_text_to_image_prompt(
assert adapt_text_to_image_prompt(
"BagelPipeline",
prompt="a cat",
negative_prompt="blurry",
{
"prompt": "a cat",
"modalities": ["image"],
"negative_prompt": "blurry",
},
height=512,
width=768,
) == {
@@ -580,13 +468,16 @@ def test_bagel_image_to_image_prompt_builder() -> None:
@pytest.mark.core_model
@pytest.mark.cpu
def test_unknown_pipeline_preserves_canonical_text_to_image_prompt() -> None:
assert build_text_to_image_prompt(
"UnknownPipeline",
prompt="a cat",
negative_prompt=None,
height=512,
width=512,
) == {"prompt": "a cat", "modalities": ["image"]}
canonical_prompt = {"prompt": "a cat", "modalities": ["image"]}
assert (
adapt_text_to_image_prompt(
"UnknownPipeline",
canonical_prompt,
height=512,
width=512,
)
is canonical_prompt
)
@pytest.mark.core_model
@@ -606,11 +497,13 @@ def test_unknown_pipeline_uses_default_image_to_image_prompt() -> None:
def _build_vace_prompt(media_inputs: dict[str, Any], *, num_frames: int = 5) -> dict:
return build_image_to_video_prompt(
return adapt_image_to_video_prompt(
"WanVACEPipeline",
prompt="a bird flying",
negative_prompt=None,
media_inputs=media_inputs,
{
"prompt": "a bird flying",
"modalities": ["video"],
"multi_modal_data": media_inputs,
},
height=16,
width=320,
num_frames=num_frames,
@@ -698,10 +591,9 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N
assert get_extra_output_params(model_class_name) == frozenset()
assert should_init_extra_args_for_non_diffusion_stages(model_class_name) is True
wrapper_prompt = build_text_to_image_prompt(
wrapper_prompt = adapt_text_to_image_prompt(
"MammothModa2ForConditionalGeneration",
prompt="a cat",
negative_prompt=None,
{"prompt": "a cat", "modalities": ["image"]},
height=256,
width=256,
)
@@ -713,10 +605,13 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N
def test_mammothmoda2_text_to_image_prompt_builder() -> None:
# Image dims are converted to the AR grid (width/16 x height/16); the negative
# prompt is ignored (MammothModa2 t2i uses CFG, not an explicit negative path).
assert build_text_to_image_prompt(
assert adapt_text_to_image_prompt(
"MammothModa2DiTPipeline",
prompt="a cat",
negative_prompt="blurry",
{
"prompt": "a cat",
"modalities": ["image"],
"negative_prompt": "blurry",
},
height=512,
width=768,
) == {
vllm_omni/entrypoints/openai/serving_video.py
+18 -2
View File
@@ -30,6 +30,7 @@ from vllm_omni.entrypoints.openai.stage_params import (
from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request
from vllm_omni.entrypoints.openai.video_api_utils import _encode_video_bytes, encode_video_base64
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
from vllm_omni.model_extras import adapt_image_to_video_prompt
from vllm_omni.outputs.output_metadata import (
DiffusionMetadataMapping,
DiffusionMultimodalOutput,
@@ -100,11 +101,14 @@ class OmniOpenAIServingVideo:
if self._stage_configs is None and stage_configs is not None:
self._stage_configs = stage_configs
def _get_diffusion_od_config(self) -> object | None:
get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None)
return get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None)
@property
def supports_mixed_reference_inputs(self) -> bool:
"""Return whether the configured diffusion model accepts mixed refs."""
get_od_config = getattr(self._engine_client, "get_diffusion_od_config", None)
od_config = get_od_config() if callable(get_od_config) else getattr(self._engine_client, "od_config", None)
od_config = self._get_diffusion_od_config()
if od_config is None:
return False
@@ -252,6 +256,18 @@ class OmniOpenAIServingVideo:
loggable = {**loggable, **redacted}
logger.info("Applied extra_params: %s", loggable)
model_class_name = getattr(self._get_diffusion_od_config(), "model_class_name", None)
prompt = cast(
OmniTextPrompt,
adapt_image_to_video_prompt(
model_class_name=model_class_name,
prompt=prompt,
height=gen_params.height,
width=gen_params.width,
num_frames=gen_params.num_frames,
),
)
self._apply_lora(request.lora, gen_params)
logger.info(