[Misc] Address prompt adapter review feedback
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu committed
1 parent
280c4f56f6
commit
a4516ea5db
3 files changed
+87
-146
No files matched your search
@@ -22,41 +22,6 @@ from vllm_omni.model_extras import (
|
||||
)
|
||||
|
||||
|
||||
def build_text_to_image_prompt(
|
||||
model_class_name: str,
|
||||
prompt: str,
|
||||
negative_prompt: str | None,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
canonical_prompt: dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"modalities": ["image"],
|
||||
}
|
||||
if negative_prompt is not None:
|
||||
canonical_prompt["negative_prompt"] = negative_prompt
|
||||
return adapt_text_to_image_prompt(model_class_name, canonical_prompt, height, width)
|
||||
|
||||
|
||||
def build_image_to_video_prompt(
|
||||
model_class_name: str,
|
||||
prompt: str,
|
||||
negative_prompt: str | None,
|
||||
media_inputs: dict[str, Any],
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
canonical_prompt: dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"modalities": ["video"],
|
||||
"multi_modal_data": media_inputs,
|
||||
}
|
||||
if negative_prompt is not None:
|
||||
canonical_prompt["negative_prompt"] = negative_prompt
|
||||
return adapt_image_to_video_prompt(model_class_name, canonical_prompt, height, width, num_frames)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize(
|
||||
@@ -172,66 +137,6 @@ def test_lingbot_extra_registry_declares_request_params() -> None:
|
||||
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_lingbot_text_to_image_uses_canonical_prompt() -> None:
|
||||
assert build_text_to_image_prompt(
|
||||
"LingBotVideoPipeline",
|
||||
prompt="a red fox in fresh snow",
|
||||
negative_prompt="",
|
||||
height=192,
|
||||
width=320,
|
||||
) == {
|
||||
"prompt": "a red fox in fresh snow",
|
||||
"modalities": ["image"],
|
||||
"negative_prompt": "",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_lingbot_image_to_video_uses_canonical_prompt() -> None:
|
||||
image = Image.new("RGB", (320, 192), "red")
|
||||
result = build_image_to_video_prompt(
|
||||
"LingBotVideoPipeline",
|
||||
prompt="the fox looks toward the camera",
|
||||
negative_prompt=None,
|
||||
media_inputs={"image": image},
|
||||
height=192,
|
||||
width=320,
|
||||
num_frames=9,
|
||||
)
|
||||
assert result == {
|
||||
"prompt": "the fox looks toward the camera",
|
||||
"modalities": ["video"],
|
||||
"multi_modal_data": {"image": image},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize(
|
||||
"media_inputs",
|
||||
[
|
||||
{},
|
||||
{"image": "input.png"},
|
||||
{"image": [Image.new("RGB", (16, 16))]},
|
||||
{"image": Image.new("RGB", (16, 16)), "video": b"video"},
|
||||
],
|
||||
ids=["missing", "path", "multiple", "extra-modality"],
|
||||
)
|
||||
def test_lingbot_image_to_video_leaves_media_validation_to_the_pipeline(
|
||||
media_inputs: dict[str, Any],
|
||||
) -> None:
|
||||
result = build_image_to_video_prompt(
|
||||
"LingBotVideoPipeline",
|
||||
prompt="move",
|
||||
negative_prompt=None,
|
||||
media_inputs=media_inputs,
|
||||
)
|
||||
assert result["multi_modal_data"] is media_inputs
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
|
||||
@@ -376,10 +281,13 @@ def test_ming_flash_omni_declared_extra_args_route_into_sampling_params() -> Non
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_ming_flash_omni_text_to_image_prompt_builder() -> None:
|
||||
assert build_text_to_image_prompt(
|
||||
assert adapt_text_to_image_prompt(
|
||||
"MingImagePipeline",
|
||||
prompt="Please draw a cute cat.",
|
||||
negative_prompt="ugly, blurry",
|
||||
{
|
||||
"prompt": "Please draw a cute cat.",
|
||||
"modalities": ["image"],
|
||||
"negative_prompt": "ugly, blurry",
|
||||
},
|
||||
height=512,
|
||||
width=768,
|
||||
) == {
|
||||
@@ -394,10 +302,9 @@ def test_ming_flash_omni_text_to_image_prompt_builder() -> None:
|
||||
}
|
||||
# target_h/w are omitted when height/width are not supplied so the
|
||||
# pipeline's 1024x1024 default still applies.
|
||||
assert build_text_to_image_prompt(
|
||||
assert adapt_text_to_image_prompt(
|
||||
"MingImagePipeline",
|
||||
prompt="Draw a poster.",
|
||||
negative_prompt=None,
|
||||
{"prompt": "Draw a poster.", "modalities": ["image"]},
|
||||
) == {
|
||||
"prompt": "Draw a poster.",
|
||||
"modalities": ["image"],
|
||||
@@ -427,28 +334,6 @@ def test_ming_flash_omni_image_to_image_prompt_builder() -> None:
|
||||
assert "negative_prompt" not in result
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("pipeline_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
|
||||
def test_cosmos3_text_to_image_uses_canonical_prompt(pipeline_name: str) -> None:
|
||||
assert build_text_to_image_prompt(
|
||||
pipeline_name,
|
||||
prompt="a red sports car at golden hour",
|
||||
negative_prompt="blurry, distorted",
|
||||
height=1024,
|
||||
width=1024,
|
||||
) == {
|
||||
"prompt": "a red sports car at golden hour",
|
||||
"modalities": ["image"],
|
||||
"negative_prompt": "blurry, distorted",
|
||||
}
|
||||
assert build_text_to_image_prompt(
|
||||
pipeline_name,
|
||||
prompt="a red sports car",
|
||||
negative_prompt=None,
|
||||
) == {"prompt": "a red sports car", "modalities": ["image"]}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_audiox_extra_registry_declares_request_and_response_params() -> None:
|
||||
@@ -539,10 +424,13 @@ def test_unknown_pipeline_has_empty_extra_registry() -> None:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_bagel_text_to_image_prompt_builder() -> None:
|
||||
assert build_text_to_image_prompt(
|
||||
assert adapt_text_to_image_prompt(
|
||||
"BagelPipeline",
|
||||
prompt="a cat",
|
||||
negative_prompt="blurry",
|
||||
{
|
||||
"prompt": "a cat",
|
||||
"modalities": ["image"],
|
||||
"negative_prompt": "blurry",
|
||||
},
|
||||
height=512,
|
||||
width=768,
|
||||
) == {
|
||||
@@ -580,13 +468,16 @@ def test_bagel_image_to_image_prompt_builder() -> None:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_unknown_pipeline_preserves_canonical_text_to_image_prompt() -> None:
|
||||
assert build_text_to_image_prompt(
|
||||
"UnknownPipeline",
|
||||
prompt="a cat",
|
||||
negative_prompt=None,
|
||||
height=512,
|
||||
width=512,
|
||||
) == {"prompt": "a cat", "modalities": ["image"]}
|
||||
canonical_prompt = {"prompt": "a cat", "modalities": ["image"]}
|
||||
assert (
|
||||
adapt_text_to_image_prompt(
|
||||
"UnknownPipeline",
|
||||
canonical_prompt,
|
||||
height=512,
|
||||
width=512,
|
||||
)
|
||||
is canonical_prompt
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@@ -606,11 +497,13 @@ def test_unknown_pipeline_uses_default_image_to_image_prompt() -> None:
|
||||
|
||||
|
||||
def _build_vace_prompt(media_inputs: dict[str, Any], *, num_frames: int = 5) -> dict:
|
||||
return build_image_to_video_prompt(
|
||||
return adapt_image_to_video_prompt(
|
||||
"WanVACEPipeline",
|
||||
prompt="a bird flying",
|
||||
negative_prompt=None,
|
||||
media_inputs=media_inputs,
|
||||
{
|
||||
"prompt": "a bird flying",
|
||||
"modalities": ["video"],
|
||||
"multi_modal_data": media_inputs,
|
||||
},
|
||||
height=16,
|
||||
width=320,
|
||||
num_frames=num_frames,
|
||||
@@ -698,10 +591,9 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N
|
||||
assert get_extra_output_params(model_class_name) == frozenset()
|
||||
assert should_init_extra_args_for_non_diffusion_stages(model_class_name) is True
|
||||
|
||||
wrapper_prompt = build_text_to_image_prompt(
|
||||
wrapper_prompt = adapt_text_to_image_prompt(
|
||||
"MammothModa2ForConditionalGeneration",
|
||||
prompt="a cat",
|
||||
negative_prompt=None,
|
||||
{"prompt": "a cat", "modalities": ["image"]},
|
||||
height=256,
|
||||
width=256,
|
||||
)
|
||||
@@ -713,10 +605,13 @@ def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> N
|
||||
def test_mammothmoda2_text_to_image_prompt_builder() -> None:
|
||||
# Image dims are converted to the AR grid (width/16 x height/16); the negative
|
||||
# prompt is ignored (MammothModa2 t2i uses CFG, not an explicit negative path).
|
||||
assert build_text_to_image_prompt(
|
||||
assert adapt_text_to_image_prompt(
|
||||
"MammothModa2DiTPipeline",
|
||||
prompt="a cat",
|
||||
negative_prompt="blurry",
|
||||
{
|
||||
"prompt": "a cat",
|
||||
"modalities": ["image"],
|
||||
"negative_prompt": "blurry",
|
||||
},
|
||||
height=512,
|
||||
width=768,
|
||||
) == {
|
||||
|
||||
Reference in new issue
Block a user