Honor Omni stage runner selection in shared vLLM config

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-10-03 22:22:40 +08:00
1 parent b256e1ec4e
commit 9361c52de3
2 files changed
+54 -2

No files matched your search

+20 -2
View File
@@ -10,6 +10,7 @@ from functools import cached_property
import torch
from aenum import extend_enum
from vllm.config import ModelConfig as _OriginalModelConfig
from vllm.config import VllmConfig as _OriginalVllmConfig
from vllm.inputs import TokensPrompt as _OriginalTokensPrompt
from vllm.model_executor.layers.rotary_embedding import (
MRotaryEmbedding as _OriginalMRotaryEmbedding,
@@ -29,6 +30,23 @@ from vllm_omni.request import OmniRequest, OmniStreamingUpdate
_PATCH_LOGGER = logging.getLogger("vllm_omni.patch")
# Omni's deploy config selects a runner per stage. vLLM 0.31's config property
# otherwise defaults to v2 independently of the runner selected by our workers,
# making shared consumers such as InputProcessor choose the wrong API.
_runner_property = _OriginalVllmConfig.use_v2_model_runner
if not getattr(_runner_property.fget, "_omni_stage_runner", False):
def _patched_use_v2_model_runner(self, _upstream=_runner_property.fget):
# Lazy import keeps the package's patch/config initialization acyclic.
from vllm_omni.config.model import OmniModelConfig
if isinstance(self.model_config, OmniModelConfig):
return self.model_config.use_v2_model_runner
return _upstream(self)
_patched_use_v2_model_runner._omni_stage_runner = True
_OriginalVllmConfig.use_v2_model_runner = property(_patched_use_v2_model_runner)
# =============================================================================
# Patch ModelConfig.is_mm_prefix_lm to support omni-specific models
# =============================================================================
@@ -390,8 +408,8 @@ def _patch_chat_template_registry():
)
if "qwen3_omni_moe" not in _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK:
_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = (
lambda _: CHAT_TEMPLATES_DIR / "template_chatml.jinja"
_MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK["qwen3_omni_moe"] = lambda _: (
CHAT_TEMPLATES_DIR / "template_chatml.jinja"
)
except ImportError:
pass