BruceLoveDecimaland汪志鹏
adf6d0c1ea
Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Signed-off-by: liuqihao <liuqihao970610@gmail.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
89 lines
4.0 KiB
Python
89 lines
4.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DiffusionModelMetadata:
|
|
# Keep serving-facing capability metadata in a lightweight shared module so
|
|
# config/model plumbing can read it without importing concrete pipelines.
|
|
supports_multimodal_inputs: bool = False
|
|
max_multimodal_image_inputs: int | None = None
|
|
supports_mixed_reference_inputs: bool = False
|
|
attention_mask_free: bool = False
|
|
|
|
|
|
QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES = 4
|
|
# Upstream HunyuanImage-3.0 "Multi-Image Fusion" caps reference images at 3.
|
|
HUNYUAN_IMAGE3_MAX_INPUT_IMAGES = 3
|
|
# Boogu-Image editing (TI2I) supports a single reference image for now.
|
|
BOOGU_IMAGE_MAX_INPUT_IMAGES = 1
|
|
|
|
|
|
_DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = {
|
|
"QwenImageEditPlusPipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES,
|
|
),
|
|
"HunyuanImage3Pipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=HUNYUAN_IMAGE3_MAX_INPUT_IMAGES,
|
|
),
|
|
# Shared by the Base (text-to-image) and Edit (TI2I) checkpoints, which use
|
|
# the same ``BooguImagePipeline`` class. Text-to-image requests simply carry
|
|
# no reference image.
|
|
"BooguImagePipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=BOOGU_IMAGE_MAX_INPUT_IMAGES,
|
|
),
|
|
"MiniMaxH3Pipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=9,
|
|
supports_mixed_reference_inputs=True,
|
|
# H3 represents alignment padding as a second packed sequence. The
|
|
# packed TRTLLM backend consumes cu_seqlens and isolates that padding.
|
|
attention_mask_free=True,
|
|
),
|
|
# The modular alias is served by MiniMaxH3Pipeline and has the same
|
|
# Ref2VA request contract. Keep admission limits in sync with it.
|
|
"MiniMaxH3ModularPipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=9,
|
|
supports_mixed_reference_inputs=True,
|
|
),
|
|
"WanPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
|
"WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
|
"WanVACEPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
|
"WanS2VPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
|
"SanaWmPipeline": DiffusionModelMetadata(
|
|
supports_multimodal_inputs=True,
|
|
max_multimodal_image_inputs=1,
|
|
),
|
|
}
|
|
|
|
|
|
def get_diffusion_model_metadata(model_class_name: str | None) -> DiffusionModelMetadata:
|
|
# Unknown models fall back to "no special multimodal capabilities" so new
|
|
# pipelines do not accidentally inherit limits meant for other models.
|
|
if model_class_name is None:
|
|
return DiffusionModelMetadata()
|
|
metadata = _DIFFUSION_MODEL_METADATA.get(model_class_name)
|
|
if metadata is not None:
|
|
return metadata
|
|
# Some checkpoints report the HF architecture name diff from internal pipeline class name
|
|
# (e.g. HunyuanImage3ForCausalMM, WanVACEPipeline, OmniVoice ...).
|
|
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
|
|
|
|
entry = _DIFFUSION_MODELS.get(model_class_name)
|
|
if entry is not None:
|
|
# Unpack instead of indexing so a future change to the registry tuple
|
|
# shape fails loudly instead of silently reading the wrong element.
|
|
_, _, pipeline_cls_name = entry
|
|
# Note: the registry ``cls_name`` and the metadata keys are two separate
|
|
# key spaces. Aliases whose pipeline class has no metadata entry (e.g.
|
|
# Wan22VACEPipeline, OmniVoicePipeline) still fall back to the defaults;
|
|
# that is not a regression, it just means no capability override.
|
|
return _DIFFUSION_MODEL_METADATA.get(pipeline_cls_name, DiffusionModelMetadata())
|
|
return DiffusionModelMetadata()
|