[Bugfix] GLM-Image: route t2i requests through the multimodal processor (#3034) (#3189)

Signed-off-by: Piotr Tarasiewicz <ptarasiewicz@nvidia.com>

tests/inputs/test_preprocess.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for OmniInputPreprocessor._process_text routing."""
+
+import pytest
+from pytest_mock import MockerFixture
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class TestProcessTextMmProcessorKwargsRouting:
+    """Presence-based routing: an explicit empty ``mm_processor_kwargs`` dict
+    on the prompt still routes through ``_process_multimodal``. Required for
+    AR-based image generation (e.g. GLM-Image t2i) where the HF processor
+    supplies the image-generation scaffold from its own defaults and no extra
+    kwargs are needed from the caller.
+    """
+
+    @pytest.fixture
+    def preprocessor(self, mocker: MockerFixture):
+        from vllm_omni.inputs.preprocess import OmniInputPreprocessor
+
+        instance = object.__new__(OmniInputPreprocessor)
+        instance._process_multimodal = mocker.MagicMock(return_value={})
+        instance._tokenize_prompt = mocker.MagicMock(return_value=[1, 2, 3])
+        return instance
+
+    def test_empty_mm_processor_kwargs_routes_to_multimodal(self, preprocessor):
+        preprocessor._process_text({"prompt": "hello", "mm_processor_kwargs": {}})
+        assert preprocessor._process_multimodal.called
+        assert not preprocessor._tokenize_prompt.called
+
+    def test_missing_mm_processor_kwargs_routes_to_tokenize(self, preprocessor):
+        preprocessor._process_text({"prompt": "hello"})
+        assert preprocessor._tokenize_prompt.called
+        assert not preprocessor._process_multimodal.called

vllm_omni/entrypoints/openai/serving_chat.py

@@ -356,14 +356,18 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
                     tprompt["modalities"] = ["image"]
                 if negative_prompt is not None:
                     tprompt["negative_prompt"] = negative_prompt
-                # GLM-Image's _call_hf_processor expects target_h/target_w in mm_processor_kwargs
+                # Always attach mm_processor_kwargs (possibly empty) so
+                # OmniInputPreprocessor._process_text routes through the
+                # multimodal processor path. Without it, the preprocessor
+                # falls back to plain _tokenize_prompt and AR-based image-gen
+                # models like GLM-Image never see their image-generation
+                # scaffold.
                 mm_processor_kwargs: dict[str, Any] = {}
                 if height is not None:
                     mm_processor_kwargs["target_h"] = height
                 if width is not None:
                     mm_processor_kwargs["target_w"] = width
-                if mm_processor_kwargs:
-                    tprompt["mm_processor_kwargs"] = mm_processor_kwargs
+                tprompt["mm_processor_kwargs"] = mm_processor_kwargs
                 if engine_prompt_image is not None:
                     tprompt["multi_modal_data"] = engine_prompt_image
                     # Provide multi_modal_uuids so that newer vLLM versions

vllm_omni/inputs/preprocess.py

@@ -60,7 +60,11 @@ class OmniInputPreprocessor(InputPreprocessor):
             additional_information = parsed_content.get("additional_information")
             if additional_information is not None:
                 inputs["additional_information"] = additional_information
-        elif mm_processor_kwargs:
+        elif "mm_processor_kwargs" in parsed_content:
+            # Presence — not truthiness. An explicitly-set empty dict still
+            # signals "route through the multimodal processor" (needed for
+            # AR-based image-gen where the HF processor supplies its own
+            # defaults and scaffold).
             inputs = self._process_multimodal(
                 prompt_text,
                 {},