[diffusion][bugfix] Separate Diffusers hooks from model metadata (#6749)
Signed-off-by: david6666666 <530634352@qq.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
73893e73ea
commit
9a88d2d21d
11 files changed
+232
-12
No files matched your search
@@ -685,6 +685,19 @@ def resolve_model_class_name(
|
||||
return None
|
||||
|
||||
|
||||
def uses_diffusers_adapter(od_config: object) -> bool:
|
||||
"""Return whether execution uses the Diffusers adapter backend.
|
||||
|
||||
A custom pipeline takes precedence over ``diffusion_load_format`` in the
|
||||
worker, so adapter-specific behavior must only apply when no custom
|
||||
pipeline override is configured.
|
||||
"""
|
||||
return (
|
||||
getattr(od_config, "custom_pipeline_args", None) is None
|
||||
and getattr(od_config, "diffusion_load_format", "default") == "diffusers"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OmniDiffusionConfig:
|
||||
# Model and path configuration (for convenience)
|
||||
@@ -1317,10 +1330,6 @@ class OmniDiffusionConfig:
|
||||
|
||||
from vllm_omni.diffusion.utils.hf_utils import get_diffusion_model_index
|
||||
|
||||
# Default model_class_name for diffusers adapter
|
||||
if self.model_class_name is None and self.diffusion_load_format == "diffusers":
|
||||
self.model_class_name = "DiffusersAdapterPipeline"
|
||||
|
||||
assert self.model is not None
|
||||
try:
|
||||
config_dict = get_diffusion_model_index(
|
||||
@@ -1330,6 +1339,8 @@ class OmniDiffusionConfig:
|
||||
if config_dict is not None:
|
||||
if self.model_class_name is None:
|
||||
self.model_class_name = config_dict.get("_class_name", None)
|
||||
if self.model_class_name is None and self.diffusion_load_format == "diffusers":
|
||||
self.model_class_name = "DiffusersAdapterPipeline"
|
||||
self.update_multimodal_support()
|
||||
|
||||
# Skip transformer config loading for diffusers adapter
|
||||
@@ -1370,6 +1381,8 @@ class OmniDiffusionConfig:
|
||||
# Skip transformer config loading for diffusers adapter
|
||||
# (non-DiT models don't have a separate transformer folder/config)
|
||||
if self.diffusion_load_format == "diffusers":
|
||||
if self.model_class_name is None:
|
||||
self.model_class_name = "DiffusersAdapterPipeline"
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
logger.warning(
|
||||
"Could not find a valid pipeline index per Diffusers format. "
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.data import (
|
||||
DiffusionOutput,
|
||||
DiffusionRequestAbortedError,
|
||||
OmniDiffusionConfig,
|
||||
uses_diffusers_adapter,
|
||||
)
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode, is_scheduler_paged_kv_mode
|
||||
from vllm_omni.diffusion.diffusion_kv.initialization import initialize_diffusion_kv_control_plane
|
||||
@@ -157,8 +158,14 @@ def _resolve_custom_pipeline_cls(custom_pipeline_args: dict[str, Any] | None) ->
|
||||
|
||||
def supports_request_batch(od_config: OmniDiffusionConfig) -> bool:
|
||||
model_cls = _resolve_custom_pipeline_cls(getattr(od_config, "custom_pipeline_args", None))
|
||||
if model_cls is None:
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(getattr(od_config, "model_class_name", None))
|
||||
if model_cls is not None:
|
||||
return bool(getattr(model_cls, "supports_request_batch", False))
|
||||
|
||||
if uses_diffusers_adapter(od_config):
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls("DiffusersAdapterPipeline")
|
||||
return bool(getattr(model_cls, "supports_request_batch", False))
|
||||
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(getattr(od_config, "model_class_name", None))
|
||||
if model_cls is None:
|
||||
return False
|
||||
return bool(getattr(model_cls, "supports_request_batch", False))
|
||||
|
||||
@@ -29,7 +29,11 @@ class BasePipelineUtils:
|
||||
pass
|
||||
|
||||
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
||||
pass
|
||||
if sampling.enable_frame_interpolation:
|
||||
raise ValueError(
|
||||
"Frame interpolation is not supported with the Diffusers backend because "
|
||||
"Diffusers returns already postprocessed frames."
|
||||
)
|
||||
|
||||
def update_call_kwargs(
|
||||
self,
|
||||
@@ -99,6 +103,7 @@ class WanPipelineUtils(BasePipelineUtils):
|
||||
)
|
||||
|
||||
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
||||
super().validate_runtime_sampling_params(sampling)
|
||||
if sampling.boundary_ratio is not None:
|
||||
raise ValueError(
|
||||
"Boundary ratio is not supported at runtime with the diffusers backend for Wan models. Please set "
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.registry import _LazyRegisteredModel, _ModelRegistry
|
||||
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, uses_diffusers_adapter
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import DistributedVaeMixin
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig, get_sp_plan_from_model
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
@@ -725,6 +725,10 @@ def _load_process_func(od_config: OmniDiffusionConfig, func_name: str):
|
||||
|
||||
|
||||
def get_diffusion_post_process_func(od_config: OmniDiffusionConfig):
|
||||
# Keep the checkpoint's native class name for modality/capability metadata,
|
||||
# but do not run its tensor postprocessor on Diffusers' decoded outputs.
|
||||
if uses_diffusers_adapter(od_config):
|
||||
return None
|
||||
if od_config.model_class_name not in _DIFFUSION_POST_PROCESS_FUNCS:
|
||||
return None
|
||||
func_name = _DIFFUSION_POST_PROCESS_FUNCS[od_config.model_class_name]
|
||||
@@ -739,6 +743,9 @@ def get_diffusion_ir_op_priority_func(od_config: OmniDiffusionConfig):
|
||||
|
||||
|
||||
def get_diffusion_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
# The adapter translates requests to Diffusers call arguments itself.
|
||||
if uses_diffusers_adapter(od_config):
|
||||
return None
|
||||
if od_config.model_class_name not in _DIFFUSION_PRE_PROCESS_FUNCS:
|
||||
return None # Return None if no pre-processing function is registered (for backward compatibility)
|
||||
func_name = _DIFFUSION_PRE_PROCESS_FUNCS[od_config.model_class_name]
|
||||
|
||||
Reference in new issue
Block a user