diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 6bad98f79..5aa49ea6c 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -155,6 +155,7 @@ steps: source_file_dependencies: - tests/e2e/offline_inference/test_minicpmo_4_5.py - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja - vllm_omni/model_executor/models/minicpmo_4_5/ - vllm_omni/model_executor/models/step_audio2/ - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index f54a58fa4..6d8d1e8b9 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -188,6 +188,7 @@ steps: - label: "Omni · MiniCPM-o 4.5 Online Test" source_file_dependencies: - tests/e2e/online_serving/test_minicpmo_4_5.py + - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja - vllm_omni/model_executor/models/minicpmo_4_5/ - vllm_omni/model_executor/models/step_audio2/ - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/ diff --git a/pyproject.toml b/pyproject.toml index 0393c9c62..2584c8c7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ where = ["."] include = ["vllm_omni*"] [tool.setuptools.package-data] -"vllm_omni" = ["_version.py", "py.typed"] +"vllm_omni" = ["_version.py", "py.typed", "transformers_utils/chat_templates/*.jinja"] [tool.setuptools_scm] # no extra settings needed, presence enables setuptools-scm diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 86f0883f7..fb0934d4f 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -28,13 +28,13 @@ also provided. - Default deploy configs (auto-loaded by HF `model_type=minicpmo` + `hf_config.version="4.5"`): - - Default single-GPU compatibility layout (auto-loaded): + - Default single-GPU compatibility layout (auto-loaded): [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) - - Recommended 2-GPU continuous-batching layout: + - Recommended 2-GPU continuous-batching layout: [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml), - - 3-GPU layout: + - 3-GPU layout: [`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) - - 8x RTX 4090 layout: + - 8x RTX 4090 layout: [`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml) - Online example + Gradio demo: [`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/) @@ -126,9 +126,20 @@ video. Use the two-GPU profile for production throughput. ```bash vllm serve openbmb/MiniCPM-o-4_5 --omni \ --trust-remote-code \ + --chat-template vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja \ + --chat-template-content-format openai \ --host 0.0.0.0 --port 8099 ``` +Run these commands from the repository root. The bundled chat template +matches the Hugging Face `chat(omni_mode=True)` content layout: media and text +parts are concatenated without adding separators, while whitespace explicitly +included in a text part is preserved. This matters for speech generation because +an inserted newline changes the Thinker condition passed to Talker. For image +questions, place the image before the question, as in the native image-chat +layout. Continue to request `use_tts_template: true` and +`enable_thinking: false` for spoken answers. + The deploy config is auto-loaded by the model registry — no `--deploy-config` flag needed for this default single-GPU layout. @@ -271,6 +282,8 @@ uses GPU 5. GPUs 6–7 are left free. vllm serve openbmb/MiniCPM-o-4_5 --omni \ --deploy-config vllm_omni/deploy/minicpmo_4_5_8x4090.yaml \ --trust-remote-code \ + --chat-template vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja \ + --chat-template-content-format openai \ --host 0.0.0.0 --port 8099 ``` diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index 290064fc3..5417d2b16 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -24,6 +24,9 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" +_NATIVE_CHAT_TEMPLATE = ( + Path(__file__).resolve().parents[3] / "vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja" +) _CI_DEPLOY = modify_stage_config( get_deploy_config_path("minicpmo_4_5.yaml"), updates={ @@ -63,7 +66,14 @@ test_params = [ model=_MODEL, stage_config_path=_CI_DEPLOY, use_stage_cli=False, - server_args=["--trust-remote-code", "--async-chunk"], + server_args=[ + "--trust-remote-code", + "--async-chunk", + "--chat-template", + str(_NATIVE_CHAT_TEMPLATE), + "--chat-template-content-format", + "openai", + ], env_dict={ "VLLM_CONFIGURE_LOGGING": "1", "VLLM_LOGGING_CONFIG_PATH": str(_PROMPT_LOGGING_CONFIG_PATH), @@ -265,11 +275,18 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: Input Setting: stream=True """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - image_data_url=image_data_url, - content_text=get_prompt("text_image"), - ) + # Match the native image-chat layout explicitly; do not rely on the + # renderer moving media placeholders ahead of text. + messages = [ + get_system_prompt(), + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image_data_url}}, + {"type": "text", "text": get_prompt("text_image")}, + ], + }, + ] request_config = { "model": omni_server.model, diff --git a/tests/model_executor/models/minicpmo_4_5/test_chat_template.py b/tests/model_executor/models/minicpmo_4_5/test_chat_template.py new file mode 100644 index 000000000..8f1068392 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_chat_template.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""MiniCPM structured-content rendering must preserve native part boundaries.""" + +from pathlib import Path + +import pytest +from jinja2 import TemplateError +from transformers.utils.chat_template_utils import _compile_jinja_template + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +TEMPLATE = Path(__file__).resolve().parents[4] / "vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja" + + +@pytest.mark.parametrize( + "parts,expected", + [ + ([{"type": "image"}, {"type": "text", "text": "Question?"}], "(./)Question?"), + ([{"type": "text", "text": "Question?"}, {"type": "image"}], "Question?(./)"), + ([{"type": "image"}, {"type": "text", "text": "\n\nQuestion?"}], "(./)\n\nQuestion?"), + ([{"type": "text", "text": " A "}, {"type": "text", "text": " B "}], " A B "), + ([{"type": "audio"}, {"type": "text", "text": "Transcribe."}], "()Transcribe."), + ([{"type": "video"}, {"type": "image"}], "()(./)"), + ("\nKeep this whitespace.\n", "\nKeep this whitespace.\n"), + ], +) +def test_native_content_concatenation(parts, expected): + rendered = _compile_jinja_template(TEMPLATE.read_text()).render( + messages=[{"role": "user", "content": parts}], + add_generation_prompt=True, + enable_thinking=False, + use_tts_template=True, + ) + assert rendered == ( + f"<|im_start|>user\n{expected}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n<|tts_bos|>" + ) + + +def test_unsupported_part_is_not_silently_discarded(): + with pytest.raises(TemplateError, match="Unsupported MiniCPM-o content part"): + _compile_jinja_template(TEMPLATE.read_text()).render( + messages=[{"role": "user", "content": [{"type": "unknown"}]}], + add_generation_prompt=True, + ) diff --git a/vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja b/vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja new file mode 100644 index 000000000..2f2bc9839 --- /dev/null +++ b/vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja @@ -0,0 +1,116 @@ +{# SPDX-License-Identifier: Apache-2.0 + Adapted from OpenBMB/MiniCPM-o-4_5 tokenizer_config.json. + Match chat(omni_mode=True): concatenate structured content parts without + injecting separators, preserving whitespace explicitly supplied as text. +#} +{%- set normalized = namespace(messages=[]) -%} +{%- for message in messages -%} + {%- if message.content is string or message.content is none -%} + {%- set normalized.messages = normalized.messages + [message] -%} + {%- else -%} + {%- set content = namespace(text='') -%} + {%- for part in message.content -%} + {%- if part.type == 'text' -%} + {%- set content.text = content.text + part.text -%} + {%- elif part.type == 'image' -%} + {%- set content.text = content.text + '(./)' -%} + {%- elif part.type == 'audio' -%} + {%- set content.text = content.text + '()' -%} + {%- elif part.type == 'video' -%} + {%- set content.text = content.text + '()' -%} + {%- else -%} + {{- raise_exception('Unsupported MiniCPM-o content part: ' + part.type) -}} + {%- endif -%} + {%- endfor -%} + {%- set normalized.messages = normalized.messages + [dict(message, content=content.text)] -%} + {%- endif -%} +{%- endfor -%} +{%- set messages = normalized.messages -%} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} + {%- if use_tts_template is defined and use_tts_template is true %} + {{- '<|tts_bos|>' }} + {%- endif %} +{%- endif -%}