Fix hunyuan resolve stop token ids (#3896)

Signed-off-by: zzh <943967662@qq.com>
This commit is contained in:
zzh authored and GitHub committed 2026-05-31 07:47:36 +08:00
1 parent 11c4fcedd3
commit ffd9771c54
5 files changed
+199 -3

No files matched your search

@@ -124,6 +124,58 @@ def test_resolve_stop_token_ids_image_tasks_stop_on_ratio_range():
assert resolve_stop_token_ids(task="i2t", bot_task=None, tokenizer=tok) == [answer_id]
assert resolve_stop_token_ids(task="t2t", bot_task=None, tokenizer=tok) == [answer_id]
# image_size="auto" is equivalent to None (need_ratio=True).
for bot in ("think", "recaption", "think_recaption", "vanilla"):
assert resolve_stop_token_ids(task="t2i", bot_task=bot, tokenizer=tok, image_size="auto") == expected
assert resolve_stop_token_ids(task="it2i", bot_task=bot, tokenizer=tok, image_size="auto") == expected
def test_resolve_stop_token_ids_explicit_image_size_stops_on_terminator():
"""When image_size is an explicit "WxH" string, need_ratio=False: AR
already knows the size and stops at the cot terminator instead of the
ratio range.
Mirrors upstream logic where ``need_ratio`` is false when the user
supplies a target size, so ``final_stop_tokens`` is set to the
recaption/think terminator (not the ratio tokens).
"""
tok = FakeTokenizer()
end_of_think_id = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</think>"]
end_of_recaption_id = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</recaption>"]
# think_recaption / recaption -> [end_of_recaption]
for bot in ("think_recaption", "recaption"):
for task in ("t2i", "it2i"):
result = resolve_stop_token_ids(task=task, bot_task=bot, tokenizer=tok, image_size="1024x1024")
assert result == [end_of_recaption_id], (
f"task={task}, bot_task={bot}, image_size='1024x1024': expected [{end_of_recaption_id}], got {result}"
)
# think -> [end_of_think, end_of_recaption]
for task in ("t2i", "it2i"):
result = resolve_stop_token_ids(task=task, bot_task="think", tokenizer=tok, image_size="1024x1024")
assert result == [end_of_think_id, end_of_recaption_id], (
f"task={task}, bot_task='think', image_size='1024x1024': "
f"expected [{end_of_think_id}, {end_of_recaption_id}], got {result}"
)
# vanilla -> no early return, falls through to ratio range
start = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_0>"]
end = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_32>"]
other_start = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_33>"]
other_end = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<img_ratio_36>"]
expected_ratio = list(range(start, end + 1)) + list(range(other_start, other_end + 1))
for task in ("t2i", "it2i"):
result = resolve_stop_token_ids(task=task, bot_task="vanilla", tokenizer=tok, image_size="1024x1024")
assert result == expected_ratio, (
f"task={task}, bot_task='vanilla', image_size='1024x1024': expected ratio range, got {result}"
)
# Text tasks are unaffected by image_size.
answer_id = HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["<answer>"]
assert resolve_stop_token_ids(task="i2t", bot_task=None, tokenizer=tok, image_size="1024x1024") == [answer_id]
@pytest.mark.parametrize(
"task,bot_task",