diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 99f7d925a..a9251a8d8 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -32,7 +32,7 @@ from starlette.routing import Route from starlette.types import ASGIApp, Receive, Scope, Send from vllm.engine.protocol import EngineClient from vllm.entrypoints.anthropic.serving import AnthropicServingMessages -from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.chat_utils import ChatTemplateConfig, load_chat_template from vllm.entrypoints.launcher import serve_http, terminate_if_errored from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer from vllm.entrypoints.openai.api_server import build_app as build_openai_app @@ -830,6 +830,12 @@ async def omni_init_app_state( if tokenizer is None or getattr(tokenizer, "chat_template", None) is None: resolved_chat_template = _load_model_chat_template_json(args.model) + chat_template_config = ChatTemplateConfig( + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + ) + if args.tool_server == "demo": tool_server: ToolServer | None = DemoToolServer() assert isinstance(tool_server, DemoToolServer) @@ -978,9 +984,7 @@ async def omni_init_app_state( state.openai_serving_models, supported_tasks=tuple(supported_tasks), request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if any(task in POOLING_TASKS for task in supported_tasks) else None @@ -990,9 +994,7 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if "embed" in supported_tasks else None @@ -1002,9 +1004,7 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - trust_request_chat_template=args.trust_request_chat_template, + chat_template_config=chat_template_config, ) if "classify" in supported_tasks else None @@ -1013,8 +1013,9 @@ async def omni_init_app_state( ServingScores( engine_client, state.openai_serving_models, + supported_tasks=tuple(supported_tasks), request_logger=request_logger, - score_template=resolved_chat_template, + chat_template_config=chat_template_config, log_error_stack=args.log_error_stack, ) if any(t in supported_tasks for t in ("embed", "score", "token_embed"))