[Hardware] Extend diffusion engine plugin extensibility for out-of-tree hardware backends (#3239)
Signed-off-by: Yuchen Jiang <yucjiang@amazon.com> Co-authored-by: Yuchen Jiang <yucjiang@amazon.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
4 files changed
+196
-3
No files matched your search
@@ -487,10 +487,65 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
|
||||
}
|
||||
|
||||
|
||||
def register_diffusion_model(
|
||||
model_arch: str,
|
||||
module_name: str,
|
||||
class_name: str,
|
||||
pre_process_func_name: str | None = None,
|
||||
post_process_func_name: str | None = None,
|
||||
) -> None:
|
||||
"""Register a diffusion model pipeline from an out-of-tree plugin.
|
||||
|
||||
This can be used to add new model architectures or to replace an
|
||||
existing built-in pipeline with a platform-optimised implementation
|
||||
(same ``model_arch`` key).
|
||||
|
||||
Args:
|
||||
model_arch: Architecture name (e.g. ``"WanPipeline"``).
|
||||
module_name: Fully qualified module path
|
||||
(e.g. ``"my_plugin.diffusion.pipeline_wan"``).
|
||||
class_name: Class name within *module_name*.
|
||||
pre_process_func_name: Optional name of the pre-process function
|
||||
located in *module_name*. Pass ``None`` to keep the existing
|
||||
entry when replacing a built-in model.
|
||||
post_process_func_name: Optional name of the post-process function
|
||||
located in *module_name*. Pass ``None`` to keep the existing
|
||||
entry when replacing a built-in model.
|
||||
"""
|
||||
# Register model class in DiffusionModelRegistry
|
||||
DiffusionModelRegistry.register_model(
|
||||
model_arch,
|
||||
f"{module_name}:{class_name}",
|
||||
)
|
||||
|
||||
# Store in _DIFFUSION_MODELS so _load_process_func can resolve the
|
||||
# module. Convention: when mod_relname is empty the mod_folder field
|
||||
# stores a *full* module path instead of a relative folder.
|
||||
_DIFFUSION_MODELS[model_arch] = (module_name, "", class_name)
|
||||
|
||||
# Optionally register pre/post process funcs.
|
||||
if pre_process_func_name is not None:
|
||||
_DIFFUSION_PRE_PROCESS_FUNCS[model_arch] = pre_process_func_name
|
||||
if post_process_func_name is not None:
|
||||
_DIFFUSION_POST_PROCESS_FUNCS[model_arch] = post_process_func_name
|
||||
|
||||
logger.info(
|
||||
"Registered diffusion model %s -> %s.%s",
|
||||
model_arch,
|
||||
module_name,
|
||||
class_name,
|
||||
)
|
||||
|
||||
|
||||
def _load_process_func(od_config: OmniDiffusionConfig, func_name: str):
|
||||
"""Load and return a process function from the appropriate module."""
|
||||
mod_folder, mod_relname, _ = _DIFFUSION_MODELS[od_config.model_class_name]
|
||||
module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
|
||||
if mod_relname == "":
|
||||
# Full module path (registered via register_diffusion_model)
|
||||
module_name = mod_folder
|
||||
else:
|
||||
# Built-in model (relative path convention)
|
||||
module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
|
||||
module = importlib.import_module(module_name)
|
||||
func = getattr(module, func_name)
|
||||
return func(od_config)
|
||||
|
||||
@@ -84,8 +84,10 @@ class DiffusionWorker:
|
||||
self.lora_manager: DiffusionLoRAManager | None = None
|
||||
self.stage_id = getattr(od_config, "stage_id", 0)
|
||||
self.init_device()
|
||||
# Create model runner
|
||||
self.model_runner = DiffusionModelRunner(
|
||||
# Create model runner using the platform-specified class
|
||||
model_runner_cls_path = current_omni_platform.get_diffusion_model_runner_cls()
|
||||
model_runner_cls = resolve_obj_by_qualname(model_runner_cls_path)
|
||||
self.model_runner = model_runner_cls(
|
||||
vllm_config=self.vllm_config,
|
||||
od_config=self.od_config,
|
||||
device=self.device,
|
||||
@@ -567,11 +569,14 @@ class WorkerProc:
|
||||
custom_pipeline_args: dict[str, Any] | None = None,
|
||||
) -> DiffusionWorker:
|
||||
"""Create a worker instance. Override in subclasses for different worker types."""
|
||||
worker_cls_path = current_omni_platform.get_diffusion_worker_cls()
|
||||
base_worker_class = resolve_obj_by_qualname(worker_cls_path)
|
||||
wrapper = WorkerWrapperBase(
|
||||
gpu_id=gpu_id,
|
||||
od_config=od_config,
|
||||
worker_extension_cls=worker_extension_cls,
|
||||
custom_pipeline_args=custom_pipeline_args,
|
||||
base_worker_class=base_worker_class,
|
||||
)
|
||||
return wrapper
|
||||
|
||||
|
||||
Reference in new issue
Block a user