From 3265e857ebde9098ffa5962f3d5e3e0773a0f6bd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 24 Oct 2025 17:15:34 +0800 Subject: [PATCH 1/5] Refactor OmniLLM and MultimodalOutputProcessor initialization - Removed AsyncOmniLLM from the main entry points as it is not supported yet. - Updated the constructor of MultimodalOutputProcessor to set a default value for engine_core_output_type. - Adjusted output type handling in various methods to utilize the new default engine_core_output_type for consistency. --- vllm_omni/__init__.py | 4 ++-- vllm_omni/engine/output_processor.py | 8 ++++---- vllm_omni/entrypoints/omni_llm.py | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index fb544b0fa..5c6cd50ba 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -14,7 +14,7 @@ __author__ = "vLLM-omni Team" __email__ = "hsliuustc@gmail.com" # Main entry points -from .entrypoints.omni_llm import OmniLLM, AsyncOmniLLM +from .entrypoints.omni_llm import OmniLLM from .config import ( OmniStageConfig, DiTConfig, @@ -31,7 +31,7 @@ __all__ = [ # Main components "OmniLLM", - "AsyncOmniLLM", + #AsyncOmniLLM is not supported yet # Configuration "OmniStageConfig", diff --git a/vllm_omni/engine/output_processor.py b/vllm_omni/engine/output_processor.py index 211db8545..3d779bb30 100644 --- a/vllm_omni/engine/output_processor.py +++ b/vllm_omni/engine/output_processor.py @@ -177,7 +177,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): produce the correct RequestOutput/PoolingRequestOutput. - Allow custom per-modality handlers via register_handler(). """ - def __init__(self, tokenizer: TokenizerGroup, log_stats: bool, engine_core_output_type: Optional[str] = None): + def __init__(self, tokenizer: TokenizerGroup, log_stats: bool, engine_core_output_type: Optional[str] = "text"): super().__init__(tokenizer=tokenizer, log_stats=log_stats) self.output_handlers: Dict[str, Callable[[EngineCoreOutput], None]] = {} self._reqid_to_mm_type: Dict[str, str] = {} @@ -223,7 +223,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): ) -> OutputProcessorOutput: self._reqid_to_mm_type.clear() for eco in engine_core_outputs: - mm_type = (getattr(eco, "output_type", None) or "").lower() + mm_type = (self.engine_core_output_type or "").lower() if mm_type: self._reqid_to_mm_type[eco.request_id] = mm_type self._route_and_normalize(eco) @@ -262,7 +262,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): # 2.5) Accumulate multimodal tensors in RequestState try: - mm_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower() + mm_type = (self.engine_core_output_type or "").lower() if pooling_output is not None and isinstance(req_state, OmniRequestState): req_state.add_multimodal_tensor(pooling_output, mm_type) except Exception: @@ -315,7 +315,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor): # ---- routing helpers ---- def _route_and_normalize(self, eco: EngineCoreOutput) -> None: - output_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower() + output_type = (self.engine_core_output_type or "").lower() # Custom handler first (if registered) if output_type in self.output_handlers: diff --git a/vllm_omni/entrypoints/omni_llm.py b/vllm_omni/entrypoints/omni_llm.py index cdfe5d53c..15fc4881e 100644 --- a/vllm_omni/entrypoints/omni_llm.py +++ b/vllm_omni/entrypoints/omni_llm.py @@ -138,7 +138,8 @@ class OmniStageLLM(LLM): self.llm_engine = LLMEngine.from_engine_args( engine_args=engine_args, usage_context=UsageContext.LLM_CLASS) self.llm_engine.output_processor = MultimodalOutputProcessor(tokenizer=self.llm_engine.tokenizer, - log_stats=self.llm_engine.log_stats) + log_stats=self.llm_engine.log_stats, + engine_core_output_type=engine_args.engine_output_type) self.llm_engine.processor = OmniProcessor(vllm_config=self.llm_engine.vllm_config, tokenizer=self.llm_engine.tokenizer) self.engine_class = type(self.llm_engine) From 61b344dc39a80c4139038c0dc14b7a9b4540b12b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 24 Oct 2025 17:44:31 +0800 Subject: [PATCH 2/5] Add import for OmniModelsRegistry in model executor initialization --- vllm_omni/model_executor/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm_omni/model_executor/models/__init__.py b/vllm_omni/model_executor/models/__init__.py index e69de29bb..e3ea88012 100644 --- a/vllm_omni/model_executor/models/__init__.py +++ b/vllm_omni/model_executor/models/__init__.py @@ -0,0 +1 @@ +from .registry import OmniModelsRegistry \ No newline at end of file From d45f8a6be71ac81ff8b4a3c3a5768cbc6112f77d Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 24 Oct 2025 17:47:49 +0800 Subject: [PATCH 3/5] debug typo --- vllm_omni/model_executor/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_omni/model_executor/models/__init__.py b/vllm_omni/model_executor/models/__init__.py index e3ea88012..27e699e12 100644 --- a/vllm_omni/model_executor/models/__init__.py +++ b/vllm_omni/model_executor/models/__init__.py @@ -1 +1 @@ -from .registry import OmniModelsRegistry \ No newline at end of file +from .registry import OmniModelRegistry \ No newline at end of file From 605d1de013ab51ad4f12d2d6ab18c55c76585cea Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 24 Oct 2025 17:54:19 +0800 Subject: [PATCH 4/5] Update entry points in vllm_omni module - Reintroduced AsyncOmniLLM to the main entry points. - Added import for patch module. - Cleaned up comments regarding AsyncOmniLLM support. --- vllm_omni/__init__.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index 4281a4e55..a8bb692c5 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -15,9 +15,7 @@ Architecture: __version__ = "0.1.0" __author__ = "vLLM-omni Team" __email__ = "hsliuustc@gmail.com" - -# Main entry points -from .entrypoints.omni_llm import OmniLLM +from . import patch from .config import ( DiTCacheConfig, DiTConfig, @@ -27,7 +25,7 @@ from .config import ( ) # Main entry points -from .entrypoints.omni_llm import OmniLLM #AsyncOmniLLM, +from .entrypoints.omni_llm import AsyncOmniLLM, OmniLLM __all__ = [ # Version info @@ -36,8 +34,7 @@ __all__ = [ "__email__", # Main components "OmniLLM", - #AsyncOmniLLM is not supported yet - + "AsyncOmniLLM", # Configuration "OmniStageConfig", "DiTConfig", From f6158e7576ab1d71f469484905a5caa6d22c19db Mon Sep 17 00:00:00 2001 From: tzhouam Date: Fri, 24 Oct 2025 18:01:02 +0800 Subject: [PATCH 5/5] debug --- vllm_omni/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index a8bb692c5..94f3cf9b8 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -25,7 +25,7 @@ from .config import ( ) # Main entry points -from .entrypoints.omni_llm import AsyncOmniLLM, OmniLLM +from .entrypoints.omni_llm import OmniLLM __all__ = [ # Version info @@ -34,7 +34,7 @@ __all__ = [ "__email__", # Main components "OmniLLM", - "AsyncOmniLLM", + # AsyncOmniLLM is not supported yet # Configuration "OmniStageConfig", "DiTConfig",