[Feature] HunyuanImage-3.0 IT2I: multi-image input + prompt API cleanup (#3444)

Signed-off-by: TaffyOfficial <2324465096@qq.com>
Signed-off-by: TaffyOfficial <wu15922848573@outlook.com>
Signed-off-by: skf1999 <13234016272@163.com>
Signed-off-by: zuiho <2324465096@qq.com>
Signed-off-by: Claude Code <noreply@anthropic.com>
Signed-off-by: zuiho <wu15922848573@outlook.com>
Signed-off-by: TaffyOfficial <2587297563@qq.com>
Co-authored-by: TaffyOfficial <2324465096@qq.com>
Co-authored-by: TaffyOfficial <wu15922848573@outlook.com>
Co-authored-by: skf1999 <13234016272@163.com>
This commit is contained in:
authored and GitHub committed 2026-05-14 16:58:40 +08:00
1 parent c4f859bf56
commit 3f63aaf982
17 files changed
+1559 -534

No files matched your search

@@ -1,20 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for HunyuanImage3 prompt construction (PR #3243).
Two layers:
1. Pure-logic tests with a recording fake tokenizer -- protect the
prompt template structure (BOS, User:/Assistant: framing, trigger
placement, image placeholder position) and protect the segment-
by-segment tokenization contract (each segment must hit
`tokenizer.encode` in isolation).
2. Real-tokenizer regression -- run when the HunyuanImage3-Instruct
tokenizer is in the local HF cache. Asserts the segment-tokenized
output diverges from the naive full-string encode, which is the
bug-tripping fixture for the cross-segment BPE merge fix
(commit 7bd429ed).
"""
from __future__ import annotations
import ast
@@ -24,7 +9,9 @@ import pathlib
import pytest
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
_TASK_PRESETS,
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS,
available_bot_tasks,
available_tasks,
build_prompt,
build_prompt_tokens,
@@ -34,18 +21,7 @@ from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# -------------------- Pure-logic structural tests --------------------
class FakeTokenizer:
"""Minimal tokenizer stub that records every encode() call.
Returns deterministic ids from convert_tokens_to_ids while
encode() returns one id per character starting at 100. This lets
tests both verify segmentation (by inspecting `encode_calls`) and
locate substrings inside the returned id list.
"""
SPECIAL = {
"<|startoftext|>": 1,
"<img>": 2,
@@ -72,85 +48,131 @@ class FakeTokenizer:
def test_available_tasks_covers_all_modalities():
tasks = set(available_tasks())
assert tasks >= {
"t2t",
"i2t",
assert set(available_tasks()) == {"t2t", "i2t", "it2i", "t2i"}
def test_available_bot_tasks_covers_all_modes():
assert set(available_bot_tasks()) == {None, "think", "recaption", "think_recaption", "vanilla"}
def test_legacy_task_presets_still_available():
assert {
"it2i_think",
"it2i_recaption",
"it2i_think_recaption",
"t2i_think",
"t2i_recaption",
"t2i_vanilla",
} <= set(_TASK_PRESETS)
def test_legacy_base_task_omitted_bot_task_keeps_plain_mode():
prompt = build_prompt("HELLO", task="i2t")
assert prompt.endswith("Assistant: ")
assert not prompt.endswith("<think>")
result = build_prompt_tokens("hi", FakeTokenizer(), task="i2t")
assert result.system_prompt_type == "en_unified"
assert result.token_ids[-1] not in {
FakeTokenizer.SPECIAL["<think>"],
FakeTokenizer.SPECIAL["<recaption>"],
}
def test_resolve_stop_token_ids_uses_answer_for_generation_tasks():
def test_legacy_composite_task_with_none_bot_task_keeps_encoded_mode():
prompt = build_prompt("HELLO", task="it2i_think", bot_task=None)
assert prompt.endswith("Assistant: <think>")
result = build_prompt_tokens("hi", FakeTokenizer(), task="it2i_recaption", bot_task=None)
assert result.token_ids[-1] == FakeTokenizer.SPECIAL["<recaption>"]
def test_default_prompt_still_uses_it2i_think_mode():
prompt = build_prompt("HELLO")
assert prompt.endswith("Assistant: <think>")
result = build_prompt_tokens("hi", FakeTokenizer())
assert result.system_prompt_type == "en_unified"
assert result.token_ids[-1] == FakeTokenizer.SPECIAL["<think>"]
def test_resolve_stop_token_ids_image_tasks_stop_on_ratio_range():
"""Image-output tasks stop on any ``<img_ratio_*>`` token.
Mirrors upstream ``modeling_hunyuan_image_3.py::generate_image``
(line 3289-3303): when ``need_ratio`` is true,
``final_stop_tokens = list(range(start_ratio, end_ratio + 1)) +
ratio_token_other_slices``. AR stops AT the ratio token sampled
after ``<img_size_*>``; the bridge then strips the trailing ratio
token before passing the cot to DiT.
"""
tok = FakeTokenizer()
start = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_0>"]
end = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_32>"]
other_start = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_33>"]
other_end = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_36>"]
expected = list(range(start, end + 1)) + list(range(other_start, other_end + 1))
# Image-output: t2i / it2i stop on the full ratio token range.
for bot in ("think", "recaption", "think_recaption", "vanilla"):
assert resolve_stop_token_ids(task="t2i", bot_task=bot, tokenizer=tok) == expected
assert resolve_stop_token_ids(task="it2i", bot_task=bot, tokenizer=tok) == expected
# Text-output: i2t / t2t comprehension stops on <answer> (response sits inside).
answer_id = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<answer>"]
assert resolve_stop_token_ids(task="t2i_think", tokenizer=tok) == [answer_id]
assert resolve_stop_token_ids(task="t2i_recaption", tokenizer=tok) == [answer_id]
assert resolve_stop_token_ids(task="i2t", bot_task=None, tokenizer=tok) == [answer_id]
assert resolve_stop_token_ids(task="t2t", bot_task=None, tokenizer=tok) == [answer_id]
@pytest.mark.parametrize(
"task",
"task,bot_task",
[
"t2t",
"i2t",
"it2i_think",
"it2i_recaption",
"it2i_think_recaption",
"t2i_think",
"t2i_recaption",
("t2t", None),
("i2t", None),
("it2i", "think"),
("it2i", "recaption"),
("it2i", "think_recaption"),
("t2i", "think"),
("t2i", "recaption"),
("t2i", "think_recaption"),
],
)
def test_build_prompt_string_structure_chat_template(task: str):
"""Chat-template tasks must produce <|startoftext|>...User: ...Assistant: ...
with image placeholder (when applicable) and trigger tag AFTER `Assistant: `."""
s = build_prompt("HELLO", task=task)
def test_build_prompt_string_structure_chat_template(task: str, bot_task: str | None):
s = build_prompt("HELLO", task=task, bot_task=bot_task)
assert s.startswith("<|startoftext|>")
assert "User: " in s
assert "Assistant: " in s
assert s.index("User: ") < s.index("HELLO") < s.index("Assistant: ")
if task.startswith(("i2t", "it2i")):
assert s.index("User: ") < s.index("<img>") < s.index("HELLO"), (
"<img> placeholder must sit between `User: ` and the user prompt"
)
if task in ("i2t", "it2i"):
assert s.index("User: ") < s.index("<img>") < s.index("HELLO")
else:
assert "<img>" not in s
# Trigger tag must be the FINAL token of the prompt (after `Assistant: `).
# Note: the system prompt itself mentions <think>/<recaption> as mode
# documentation, so substring index() catches the wrong occurrence -- use
# endswith() which directly captures "trigger is at the tail" (the Part A
# fix: trigger goes AFTER `Assistant: `, not before user_prompt).
if task in ("it2i_think", "t2i_think", "it2i_think_recaption"):
assert s.endswith("Assistant: <think>"), (
f"Trigger <think> must be appended right after `Assistant: ` (Part A fix). Got tail: ...{s[-40:]!r}"
)
if task in ("it2i_recaption", "t2i_recaption"):
assert s.endswith("Assistant: <recaption>"), (
f"Trigger <recaption> must be appended right after `Assistant: ` (Part A fix). Got tail: ...{s[-40:]!r}"
)
if task in ("t2t", "i2t"):
assert s.endswith("Assistant: "), "Plain (no-trigger) task must end at `Assistant: ` with no trailing tag."
if bot_task in ("think", "think_recaption"):
assert s.endswith("Assistant: <think>")
elif bot_task == "recaption":
assert s.endswith("Assistant: <recaption>")
elif bot_task is None:
assert s.endswith("Assistant: ")
def test_build_prompt_vanilla_uses_pretrain_template():
"""t2i_vanilla is the only task that bypasses chat structure -- direct
text->image generation driven by the vanilla system prompt."""
s = build_prompt("HELLO", task="t2i_vanilla")
s = build_prompt("HELLO", task="t2i", bot_task="vanilla")
assert s.startswith("<|startoftext|>")
assert "User: " not in s
assert "Assistant: " not in s
assert "<think>" not in s
assert "<recaption>" not in s
assert s.endswith("HELLO")
def test_build_prompt_vanilla_rejects_non_t2i_task():
with pytest.raises(ValueError, match="bot_task='vanilla'"):
build_prompt("x", task="it2i", bot_task="vanilla")
with pytest.raises(ValueError, match="bot_task='vanilla'"):
build_prompt_tokens("x", FakeTokenizer(), task="i2t", bot_task="vanilla")
def test_build_prompt_unknown_task_raises():
with pytest.raises(ValueError, match="Unknown task"):
build_prompt("x", task="bogus")
@@ -158,127 +180,83 @@ def test_build_prompt_unknown_task_raises():
build_prompt_tokens("x", FakeTokenizer(), task="bogus")
def test_build_prompt_unknown_bot_task_raises():
with pytest.raises(ValueError, match="Unknown bot_task"):
build_prompt("x", task="t2i", bot_task="bogus")
with pytest.raises(ValueError, match="Unknown bot_task"):
build_prompt_tokens("x", FakeTokenizer(), task="t2i", bot_task="bogus")
def test_build_prompt_tokens_segments_each_boundary():
"""Regression for cross-segment BPE merge bug (commit 7bd429ed):
each template segment must hit tokenizer.encode() independently;
user_prompt MUST NOT be concatenated with the following separator
in the same encode() call."""
tok = FakeTokenizer()
build_prompt_tokens("写诗。", tok, task="i2t")
# Each canonical segment is encoded in its own call.
build_prompt_tokens("写诗。", tok, task="i2t", bot_task=None)
assert "User: " in tok.encode_calls
assert "写诗。" in tok.encode_calls, (
"user_prompt must be encoded alone -- if it is concatenated with the "
"trailing separator, BPE will merge across the boundary (the PR-#3243 bug)."
)
assert "写诗。" in tok.encode_calls
assert "\n\nAssistant: " in tok.encode_calls
# No call must contain user_prompt glued to neighboring text.
for call in tok.encode_calls:
if call != "写诗。":
assert "写诗。" not in call, f"user_prompt leaked into a multi-segment encode call: {call!r}"
assert "写诗。" not in call
def test_build_prompt_tokens_image_placeholder_present_for_image_tasks():
tok = FakeTokenizer()
result = build_prompt_tokens("hi", tok, task="i2t")
result = build_prompt_tokens("hi", tok, task="i2t", bot_task=None)
ids = result.token_ids
assert ids[0] == FakeTokenizer.SPECIAL["<|startoftext|>"], "BOS (<|startoftext|>) must be the first token"
assert FakeTokenizer.SPECIAL["<img>"] in ids, "<img> placeholder must be present for i2t/it2i tasks"
assert ids[0] == FakeTokenizer.SPECIAL["<|startoftext|>"]
assert FakeTokenizer.SPECIAL["<img>"] in ids
def test_build_prompt_tokens_no_image_for_text_only_tasks():
tok = FakeTokenizer()
result = build_prompt_tokens("hi", tok, task="t2t")
result = build_prompt_tokens("hi", tok, task="t2t", bot_task=None)
ids = result.token_ids
assert FakeTokenizer.SPECIAL["<img>"] not in ids, "<img> must NOT appear for text-only tasks"
assert FakeTokenizer.SPECIAL["<img>"] not in ids
@pytest.mark.parametrize(
"task,trigger_id",
"task,bot_task,trigger_id",
[
("it2i_think", FakeTokenizer.SPECIAL["<think>"]),
("t2i_think", FakeTokenizer.SPECIAL["<think>"]),
("it2i_recaption", FakeTokenizer.SPECIAL["<recaption>"]),
("t2i_recaption", FakeTokenizer.SPECIAL["<recaption>"]),
("it2i", "think", FakeTokenizer.SPECIAL["<think>"]),
("t2i", "think", FakeTokenizer.SPECIAL["<think>"]),
("t2i", "think_recaption", FakeTokenizer.SPECIAL["<think>"]),
("it2i", "recaption", FakeTokenizer.SPECIAL["<recaption>"]),
("t2i", "recaption", FakeTokenizer.SPECIAL["<recaption>"]),
("it2i_think", None, FakeTokenizer.SPECIAL["<think>"]),
("it2i_recaption", None, FakeTokenizer.SPECIAL["<recaption>"]),
],
)
def test_build_prompt_tokens_trigger_is_last_token(task: str, trigger_id: int):
"""Trigger tag id must be the LAST token (after `Assistant: ` segment)."""
def test_build_prompt_tokens_trigger_is_last_token(task: str, bot_task: str | None, trigger_id: int):
tok = FakeTokenizer()
result = build_prompt_tokens("hi", tok, task=task)
ids = result.token_ids
assert ids[-1] == trigger_id
result = build_prompt_tokens("hi", tok, task=task, bot_task=bot_task)
assert result.token_ids[-1] == trigger_id
def test_build_prompt_tokens_no_trigger_for_plain_tasks():
"""Tasks without trigger_tag (t2t / i2t) must NOT append a trigger id."""
tok = FakeTokenizer()
result = build_prompt_tokens("hi", tok, task="t2t")
ids = result.token_ids
assert ids[-1] not in {
result = build_prompt_tokens("hi", tok, task="t2t", bot_task=None)
assert result.token_ids[-1] not in {
FakeTokenizer.SPECIAL["<think>"],
FakeTokenizer.SPECIAL["<recaption>"],
}
# -------------------- end2end.py wiring guard --------------------
def _repo_root() -> pathlib.Path:
# tests/diffusion/models/hunyuan_image3/test_prompt_utils.py -> repo root
return pathlib.Path(__file__).resolve().parents[4]
def test_end2end_routes_through_shared_prompt_utils():
"""Regression for the *delivery vector* of PR #3243.
Background: the wrong-template bug that PR #3243 fixes was introduced
when end2end.py grew its own hand-rolled prompt builder that diverged
from the canonical instruct chat template. To prevent that exact
failure mode from recurring, end2end.py MUST:
1. Import the prompt builders from the shared prompt_utils module.
2. NOT redefine `build_prompt` or `build_prompt_tokens` locally.
A local redefinition is precisely how a future merge can silently
re-introduce a pretrain-style template (trigger BEFORE user_prompt,
no User:/Assistant: framing, etc.) without touching prompt_utils,
bypassing every other test in this file.
"""
end2end_path = _repo_root() / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"
assert end2end_path.is_file(), f"end2end.py not found at {end2end_path}"
tree = ast.parse(end2end_path.read_text(encoding="utf-8"))
local_func_names = {n.name for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)}
forbidden = {"build_prompt", "build_prompt_tokens"}
redefined = local_func_names & forbidden
assert not redefined, (
f"end2end.py defines {sorted(redefined)} locally. This is exactly how "
"the wrong prompt template re-entered the example before PR #3243. "
"Use the shared `vllm_omni.diffusion.models.hunyuan_image3.prompt_utils` "
"helpers instead."
)
assert not (local_func_names & {"build_prompt", "build_prompt_tokens"})
imported_from_prompt_utils: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module and node.module.endswith("hunyuan_image3.prompt_utils"):
imported_from_prompt_utils.update(alias.name for alias in node.names)
expected_imports = {
"_TASK_PRESETS",
"build_prompt_tokens",
"resolve_stop_token_ids",
}
assert expected_imports <= imported_from_prompt_utils, (
"end2end.py must import the HunyuanImage3 prompt and stop-token helpers from "
"vllm_omni.diffusion.models.hunyuan_image3.prompt_utils -- the shared "
"module is the single source of truth for the AR-prefill template and "
"bot_task-derived AR stop token ids."
)
# -------------------- Real-tokenizer regression --------------------
expected_imports = {"build_prompt_tokens", "resolve_stop_token_ids", "resolve_sys_type"}
assert expected_imports <= imported_from_prompt_utils
_HUNYUAN_MODEL_ID = "tencent/HunyuanImage-3.0-Instruct"
@@ -290,41 +268,14 @@ def _hf_cached(model_id: str) -> bool:
return os.path.isdir(snap_dir) and any(os.scandir(snap_dir))
@pytest.mark.skipif(
not _hf_cached(_HUNYUAN_MODEL_ID),
reason=f"{_HUNYUAN_MODEL_ID} tokenizer not in HF cache",
)
@pytest.mark.skipif(not _hf_cached(_HUNYUAN_MODEL_ID), reason=f"{_HUNYUAN_MODEL_ID} tokenizer not in HF cache")
def test_segment_tokenize_diverges_from_full_string_encode():
"""Regression for PR #3243 segment-tokenization fix.
The naive `tokenizer.encode(build_prompt(...))` lets BPE merge tokens
across segment boundaries (notably `。\\n\\n` -> a single id), which
drifts the AR prefill away from HF's apply_chat_template output. The
segment-by-segment build_prompt_tokens must produce a STRICTLY
DIFFERENT id sequence on a prompt that triggers the merge.
If someone "simplifies" build_prompt_tokens to call encode() on the
full string, this assertion fires.
"""
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(_HUNYUAN_MODEL_ID, trust_remote_code=True)
user_prompt = "写一首关于夜的诗。"
result = build_prompt_tokens(user_prompt, tok, task="i2t")
result = build_prompt_tokens(user_prompt, tok, task="i2t", bot_task=None)
seg_ids = result.token_ids
full_ids = tok.encode(build_prompt(user_prompt, task="i2t"), add_special_tokens=False)
assert seg_ids != full_ids, (
"build_prompt_tokens output equals naive full-string encode -- "
"the BPE-merge-bypass behavior is no longer exercised. This means "
"the segment-by-segment fix from PR #3243 has been silently undone."
)
# Segmenting prevents merges, so the segment id list should have AT LEAST
# as many tokens as the merged version (a merge consumes 2+ ids -> 1).
assert len(seg_ids) >= len(full_ids), (
f"segment-encoded length ({len(seg_ids)}) shorter than full-string "
f"merged length ({len(full_ids)}) -- impossible if segmentation is "
f"genuinely bypassing merges."
)
full_ids = tok.encode(build_prompt(user_prompt, task="i2t", bot_task=None), add_special_tokens=False)
assert seg_ids != full_ids
assert len(seg_ids) >= len(full_ids)