From f18bb69b7ddac5b2fe73e28c38c7fbe26a8da9cb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Sat, 11 Jul 2026 00:46:45 +0800 Subject: [PATCH] fix(entrypoints): pass ChatTemplateConfig to pooling serving classes for vLLM 0.25 vLLM 0.25 reworked the pooling serving base: PoolingServing.__init__ now requires a `chat_template_config: ChatTemplateConfig` keyword and no longer accepts the old `chat_template` / `chat_template_content_format` / `trust_request_chat_template` kwargs. Build a ChatTemplateConfig once and pass it to the embedding / pooling / classification / score / responses serving objects. ServingTokenization keeps the flat kwargs (unchanged upstream). Without this, omni_init_app_state crashes at startup (missing required arg / unexpected kwargs) whenever an embed/classify/score task is served. Co-Authored-By: Claude Opus 4.8 (1M context) --- vllm_omni/entrypoints/openai/api_server.py | 23 +++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 99f7d925a..a9251a8d8 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -32,7 +32,7 @@ from starlette.routing import Route from starlette.types import ASGIApp, Receive, Scope, Send from vllm.engine.protocol import EngineClient from vllm.entrypoints.anthropic.serving import AnthropicServingMessages -from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.chat_utils import ChatTemplateConfig, load_chat_template from vllm.entrypoints.launcher import serve_http, terminate_if_errored from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer from vllm.entrypoints.openai.api_server import build_app as build_openai_app @@ -830,6 +830,12 @@ async def omni_init_app_state( if tokenizer is None or getattr(tokenizer, "chat_template", None) is None: resolved_chat_template = _load_model_chat_template_json(args.model) + chat_template_config = ChatTemplateConfig( + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + ) + if args.tool_server == "demo": tool_server: ToolServer | None = DemoToolServer() assert isinstance(tool_server, DemoToolServer) @@ -978,9 +984,7 @@ async def omni_init_app_state( state.openai_serving_models, supported_tasks=tuple(supported_tasks), request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if any(task in POOLING_TASKS for task in supported_tasks) else None @@ -990,9 +994,7 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if "embed" in supported_tasks else None @@ -1002,9 +1004,7 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if "classify" in supported_tasks else None @@ -1013,8 +1013,9 @@ async def omni_init_app_state( ServingScores( engine_client, state.openai_serving_models, + supported_tasks=tuple(supported_tasks), request_logger=request_logger, - score_template=resolved_chat_template, + chat_template_config=chat_template_config, log_error_stack=args.log_error_stack, ) if any(t in supported_tasks for t in ("embed", "score", "token_embed"))