anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(entrypoints): pass ChatTemplateConfig to pooling serving classes for vLLM 0.25

Browse Source 
vLLM 0.25 reworked the pooling serving base: PoolingServing.__init__ now
requires a `chat_template_config: ChatTemplateConfig` keyword and no longer
accepts the old `chat_template` / `chat_template_content_format` /
`trust_request_chat_template` kwargs. Build a ChatTemplateConfig once and pass
it to the embedding / pooling / classification / score / responses serving
objects. ServingTokenization keeps the flat kwargs (unchanged upstream).

Without this, omni_init_app_state crashes at startup (missing required arg /
unexpected kwargs) whenever an embed/classify/score task is served.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Opus 4.8 committed 2026-07-11 14:21:18 +08:00
1 parent fe16565cf6
commit f18bb69b7d
1 file changed
+12 -11
Download Patch File Download Diff File Expand all files Collapse all files
vllm_omni/entrypoints/openai/api_server.py
+12 -11
View File
@@ -32,7 +32,7 @@ from starlette.routing import Route
from starlette.types import ASGIApp, Receive, Scope, Send
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.chat_utils import ChatTemplateConfig, load_chat_template
from vllm.entrypoints.launcher import serve_http, terminate_if_errored
from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer
from vllm.entrypoints.openai.api_server import build_app as build_openai_app
@@ -830,6 +830,12 @@ async def omni_init_app_state(
if tokenizer is None or getattr(tokenizer, "chat_template", None) is None:
resolved_chat_template = _load_model_chat_template_json(args.model)
chat_template_config = ChatTemplateConfig(
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
)
if args.tool_server == "demo":
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
@@ -978,9 +984,7 @@ async def omni_init_app_state(
state.openai_serving_models,
supported_tasks=tuple(supported_tasks),
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
chat_template_config=chat_template_config,
)
if any(task in POOLING_TASKS for task in supported_tasks)
else None
@@ -990,9 +994,7 @@ async def omni_init_app_state(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
chat_template_config=chat_template_config,
)
if "embed" in supported_tasks
else None
@@ -1002,9 +1004,7 @@ async def omni_init_app_state(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
chat_template_config=chat_template_config,
)
if "classify" in supported_tasks
else None
@@ -1013,8 +1013,9 @@ async def omni_init_app_state(
ServingScores(
engine_client,
state.openai_serving_models,
supported_tasks=tuple(supported_tasks),
request_logger=request_logger,
score_template=resolved_chat_template,
chat_template_config=chat_template_config,
log_error_stack=args.log_error_stack,
)
if any(t in supported_tasks for t in ("embed", "score", "token_embed"))