[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
139a47a578
commit
e36babd48f
26 files changed
+2071
-200
No files matched your search
@@ -5,12 +5,13 @@ A pipeline only declares the modalities that it supports.
|
||||
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
|
||||
|
||||
Outer key: pipeline architecture name (``od_config.model_class_name``).
|
||||
Inner key: modality name (e.g. ``\"prompt\"``).
|
||||
Inner key: modality name (e.g. ``\"camera\"``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import SE3DeltaCameraHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
|
||||
|
||||
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
@@ -21,4 +22,7 @@ STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
"HeliosPyramidPipeline": {
|
||||
"prompt": PromptInteractionHandler,
|
||||
},
|
||||
"LingBotWorldCausalDMDPipeline": {
|
||||
"camera": SE3DeltaCameraHandler,
|
||||
},
|
||||
}
|
||||
Reference in new issue
Block a user