[New Model] Ming-Image (inclusionAI/Ming-Image-0.1-Design) Support (#8021)
Signed-off-by: Yuanheng <jonathan.zhaoyh@gmail.com>
This commit is contained in:
1 parent
cb5f508212
commit
71125f80ad
31 files changed
+1952
-51
No files matched your search
+41
-17
@@ -1487,6 +1487,29 @@ class OmniDiffusionConfig:
|
||||
self.max_multimodal_image_inputs = metadata.max_multimodal_image_inputs
|
||||
self.supports_mixed_reference_inputs = metadata.supports_mixed_reference_inputs
|
||||
|
||||
def _load_component_transformer_config(self) -> bool:
|
||||
"""Load a registered pipeline's component-level DiT configuration."""
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
from vllm_omni.model_extras import get_transformer_config_subfolder
|
||||
|
||||
transformer_subfolder = get_transformer_config_subfolder(
|
||||
self.model_class_name,
|
||||
model=self.model,
|
||||
revision=self.revision,
|
||||
)
|
||||
tf_config_dict = get_hf_file_to_dict(
|
||||
f"{transformer_subfolder}/config.json",
|
||||
self.model,
|
||||
revision=self.revision,
|
||||
)
|
||||
if tf_config_dict is None:
|
||||
tf_config_dict = get_hf_file_to_dict("unet/config.json", self.model, revision=self.revision)
|
||||
if tf_config_dict is None:
|
||||
return False
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(tf_config_dict))
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_lance_subfolder(model: str | None) -> bool:
|
||||
"""Return True when ``--model`` points at a Lance per-component subfolder.
|
||||
@@ -1553,23 +1576,7 @@ class OmniDiffusionConfig:
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
from vllm_omni.model_extras import get_transformer_config_subfolder
|
||||
|
||||
transformer_subfolder = get_transformer_config_subfolder(
|
||||
self.model_class_name,
|
||||
model=self.model,
|
||||
revision=self.revision,
|
||||
)
|
||||
tf_config_dict = get_hf_file_to_dict(
|
||||
f"{transformer_subfolder}/config.json",
|
||||
self.model,
|
||||
revision=self.revision,
|
||||
)
|
||||
if tf_config_dict is None:
|
||||
tf_config_dict = get_hf_file_to_dict("unet/config.json", self.model, revision=self.revision)
|
||||
if tf_config_dict is not None:
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(tf_config_dict))
|
||||
else:
|
||||
if not self._load_component_transformer_config():
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
else:
|
||||
raise FileNotFoundError("Diffusers pipeline index not found")
|
||||
@@ -1605,6 +1612,23 @@ class OmniDiffusionConfig:
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
return
|
||||
# An explicit topology or CLI override can select a native,
|
||||
# registered pipeline whose repository contains component
|
||||
# configs but no root HF config or Diffusers index. Trust
|
||||
# that explicit class selection only when its component
|
||||
# transformer config is present.
|
||||
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
||||
|
||||
if (
|
||||
self.model_class_name in DiffusionModelRegistry.get_supported_archs()
|
||||
and self._load_component_transformer_config()
|
||||
):
|
||||
logger.info(
|
||||
"Using explicitly selected diffusion pipeline %r with component-level configuration.",
|
||||
self.model_class_name,
|
||||
)
|
||||
self.update_multimodal_support()
|
||||
return
|
||||
raise ValueError(f"Could not find config.json or a Diffusers pipeline index for {self.model}")
|
||||
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(cfg))
|
||||
|
||||
Reference in new issue
Block a user