[Hardware] Extend diffusion engine plugin extensibility for out-of-tree hardware backends (#3239)

Signed-off-by: Yuchen Jiang <yucjiang@amazon.com>
Co-authored-by: Yuchen Jiang <yucjiang@amazon.com>
Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-05-08 14:29:15 +08:00
1 parent 6f2ad7b403
commit ea4cf77f56
4 files changed
+196 -3

No files matched your search

+56 -1
View File
@@ -487,10 +487,65 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
}
def register_diffusion_model(
model_arch: str,
module_name: str,
class_name: str,
pre_process_func_name: str | None = None,
post_process_func_name: str | None = None,
) -> None:
"""Register a diffusion model pipeline from an out-of-tree plugin.
This can be used to add new model architectures or to replace an
existing built-in pipeline with a platform-optimised implementation
(same ``model_arch`` key).
Args:
model_arch: Architecture name (e.g. ``"WanPipeline"``).
module_name: Fully qualified module path
(e.g. ``"my_plugin.diffusion.pipeline_wan"``).
class_name: Class name within *module_name*.
pre_process_func_name: Optional name of the pre-process function
located in *module_name*. Pass ``None`` to keep the existing
entry when replacing a built-in model.
post_process_func_name: Optional name of the post-process function
located in *module_name*. Pass ``None`` to keep the existing
entry when replacing a built-in model.
"""
# Register model class in DiffusionModelRegistry
DiffusionModelRegistry.register_model(
model_arch,
f"{module_name}:{class_name}",
)
# Store in _DIFFUSION_MODELS so _load_process_func can resolve the
# module. Convention: when mod_relname is empty the mod_folder field
# stores a *full* module path instead of a relative folder.
_DIFFUSION_MODELS[model_arch] = (module_name, "", class_name)
# Optionally register pre/post process funcs.
if pre_process_func_name is not None:
_DIFFUSION_PRE_PROCESS_FUNCS[model_arch] = pre_process_func_name
if post_process_func_name is not None:
_DIFFUSION_POST_PROCESS_FUNCS[model_arch] = post_process_func_name
logger.info(
"Registered diffusion model %s -> %s.%s",
model_arch,
module_name,
class_name,
)
def _load_process_func(od_config: OmniDiffusionConfig, func_name: str):
"""Load and return a process function from the appropriate module."""
mod_folder, mod_relname, _ = _DIFFUSION_MODELS[od_config.model_class_name]
module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
if mod_relname == "":
# Full module path (registered via register_diffusion_model)
module_name = mod_folder
else:
# Built-in model (relative path convention)
module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
module = importlib.import_module(module_name)
func = getattr(module, func_name)
return func(od_config)
@@ -84,8 +84,10 @@ class DiffusionWorker:
self.lora_manager: DiffusionLoRAManager | None = None
self.stage_id = getattr(od_config, "stage_id", 0)
self.init_device()
# Create model runner
self.model_runner = DiffusionModelRunner(
# Create model runner using the platform-specified class
model_runner_cls_path = current_omni_platform.get_diffusion_model_runner_cls()
model_runner_cls = resolve_obj_by_qualname(model_runner_cls_path)
self.model_runner = model_runner_cls(
vllm_config=self.vllm_config,
od_config=self.od_config,
device=self.device,
@@ -567,11 +569,14 @@ class WorkerProc:
custom_pipeline_args: dict[str, Any] | None = None,
) -> DiffusionWorker:
"""Create a worker instance. Override in subclasses for different worker types."""
worker_cls_path = current_omni_platform.get_diffusion_worker_cls()
base_worker_class = resolve_obj_by_qualname(worker_cls_path)
wrapper = WorkerWrapperBase(
gpu_id=gpu_id,
od_config=od_config,
worker_extension_cls=worker_extension_cls,
custom_pipeline_args=custom_pipeline_args,
base_worker_class=base_worker_class,
)
return wrapper