[diffusion][bugfix] Separate Diffusers hooks from model metadata (#6749)

Signed-off-by: david6666666 <530634352@qq.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
WeiQing ChenandGao Han authored and GitHub committed 2026-08-29 17:37:14 +00:00
1 parent 73893e73ea
commit 9a88d2d21d
11 files changed
+232 -12

No files matched your search

+17 -4
View File
@@ -685,6 +685,19 @@ def resolve_model_class_name(
return None
def uses_diffusers_adapter(od_config: object) -> bool:
"""Return whether execution uses the Diffusers adapter backend.
A custom pipeline takes precedence over ``diffusion_load_format`` in the
worker, so adapter-specific behavior must only apply when no custom
pipeline override is configured.
"""
return (
getattr(od_config, "custom_pipeline_args", None) is None
and getattr(od_config, "diffusion_load_format", "default") == "diffusers"
)
@dataclass
class OmniDiffusionConfig:
# Model and path configuration (for convenience)
@@ -1317,10 +1330,6 @@ class OmniDiffusionConfig:
from vllm_omni.diffusion.utils.hf_utils import get_diffusion_model_index
# Default model_class_name for diffusers adapter
if self.model_class_name is None and self.diffusion_load_format == "diffusers":
self.model_class_name = "DiffusersAdapterPipeline"
assert self.model is not None
try:
config_dict = get_diffusion_model_index(
@@ -1330,6 +1339,8 @@ class OmniDiffusionConfig:
if config_dict is not None:
if self.model_class_name is None:
self.model_class_name = config_dict.get("_class_name", None)
if self.model_class_name is None and self.diffusion_load_format == "diffusers":
self.model_class_name = "DiffusersAdapterPipeline"
self.update_multimodal_support()
# Skip transformer config loading for diffusers adapter
@@ -1370,6 +1381,8 @@ class OmniDiffusionConfig:
# Skip transformer config loading for diffusers adapter
# (non-DiT models don't have a separate transformer folder/config)
if self.diffusion_load_format == "diffusers":
if self.model_class_name is None:
self.model_class_name = "DiffusersAdapterPipeline"
self.set_tf_model_config(TransformerConfig())
logger.warning(
"Could not find a valid pipeline index per Diffusers format. "
+9 -2
View File
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.data import (
DiffusionOutput,
DiffusionRequestAbortedError,
OmniDiffusionConfig,
uses_diffusers_adapter,
)
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode, is_scheduler_paged_kv_mode
from vllm_omni.diffusion.diffusion_kv.initialization import initialize_diffusion_kv_control_plane
@@ -157,8 +158,14 @@ def _resolve_custom_pipeline_cls(custom_pipeline_args: dict[str, Any] | None) ->
def supports_request_batch(od_config: OmniDiffusionConfig) -> bool:
model_cls = _resolve_custom_pipeline_cls(getattr(od_config, "custom_pipeline_args", None))
if model_cls is None:
model_cls = DiffusionModelRegistry._try_load_model_cls(getattr(od_config, "model_class_name", None))
if model_cls is not None:
return bool(getattr(model_cls, "supports_request_batch", False))
if uses_diffusers_adapter(od_config):
model_cls = DiffusionModelRegistry._try_load_model_cls("DiffusersAdapterPipeline")
return bool(getattr(model_cls, "supports_request_batch", False))
model_cls = DiffusionModelRegistry._try_load_model_cls(getattr(od_config, "model_class_name", None))
if model_cls is None:
return False
return bool(getattr(model_cls, "supports_request_batch", False))
@@ -29,7 +29,11 @@ class BasePipelineUtils:
pass
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
pass
if sampling.enable_frame_interpolation:
raise ValueError(
"Frame interpolation is not supported with the Diffusers backend because "
"Diffusers returns already postprocessed frames."
)
def update_call_kwargs(
self,
@@ -99,6 +103,7 @@ class WanPipelineUtils(BasePipelineUtils):
)
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
super().validate_runtime_sampling_params(sampling)
if sampling.boundary_ratio is not None:
raise ValueError(
"Boundary ratio is not supported at runtime with the diffusers backend for Wan models. Please set "
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import copy
import inspect
+8 -1
View File
@@ -9,7 +9,7 @@ from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.registry import _LazyRegisteredModel, _ModelRegistry
from vllm_omni.diffusion.config import set_current_diffusion_config
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.data import OmniDiffusionConfig, uses_diffusers_adapter
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import DistributedVaeMixin
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig, get_sp_plan_from_model
from vllm_omni.diffusion.forward_context import get_forward_context
@@ -725,6 +725,10 @@ def _load_process_func(od_config: OmniDiffusionConfig, func_name: str):
def get_diffusion_post_process_func(od_config: OmniDiffusionConfig):
# Keep the checkpoint's native class name for modality/capability metadata,
# but do not run its tensor postprocessor on Diffusers' decoded outputs.
if uses_diffusers_adapter(od_config):
return None
if od_config.model_class_name not in _DIFFUSION_POST_PROCESS_FUNCS:
return None
func_name = _DIFFUSION_POST_PROCESS_FUNCS[od_config.model_class_name]
@@ -739,6 +743,9 @@ def get_diffusion_ir_op_priority_func(od_config: OmniDiffusionConfig):
def get_diffusion_pre_process_func(od_config: OmniDiffusionConfig):
# The adapter translates requests to Diffusers call arguments itself.
if uses_diffusers_adapter(od_config):
return None
if od_config.model_class_name not in _DIFFUSION_PRE_PROCESS_FUNCS:
return None # Return None if no pre-processing function is registered (for backward compatibility)
func_name = _DIFFUSION_PRE_PROCESS_FUNCS[od_config.model_class_name]