[Bugfix] Align the AR and DiT prompt formatting across both online and offline modes. (#3516)
Signed-off-by: dengyunyang <584797741@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
ce621be29b
commit
0d91fbbbb7
5 files changed
+49
-18
No files matched your search
@@ -192,9 +192,9 @@ def main():
|
|||||||
# Format prompts
|
# Format prompts
|
||||||
formatted_prompts: list[OmniPromptType] = []
|
formatted_prompts: list[OmniPromptType] = []
|
||||||
for p in prompts:
|
for p in prompts:
|
||||||
token_ids = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
|
result = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
|
||||||
preset_sys_type, _, _ = _TASK_PRESETS[task]
|
token_ids = result.token_ids
|
||||||
effective_sys_type = args.sys_type or preset_sys_type
|
effective_sys_type = result.system_prompt_type
|
||||||
|
|
||||||
# `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
|
# `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
|
||||||
# `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
|
# `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
|
||||||
|
|||||||
@@ -180,14 +180,16 @@ def test_build_prompt_tokens_segments_each_boundary():
|
|||||||
|
|
||||||
def test_build_prompt_tokens_image_placeholder_present_for_image_tasks():
|
def test_build_prompt_tokens_image_placeholder_present_for_image_tasks():
|
||||||
tok = FakeTokenizer()
|
tok = FakeTokenizer()
|
||||||
ids = build_prompt_tokens("hi", tok, task="i2t")
|
result = build_prompt_tokens("hi", tok, task="i2t")
|
||||||
|
ids = result.token_ids
|
||||||
assert ids[0] == FakeTokenizer.SPECIAL["<|startoftext|>"], "BOS (<|startoftext|>) must be the first token"
|
assert ids[0] == FakeTokenizer.SPECIAL["<|startoftext|>"], "BOS (<|startoftext|>) must be the first token"
|
||||||
assert FakeTokenizer.SPECIAL["<img>"] in ids, "<img> placeholder must be present for i2t/it2i tasks"
|
assert FakeTokenizer.SPECIAL["<img>"] in ids, "<img> placeholder must be present for i2t/it2i tasks"
|
||||||
|
|
||||||
|
|
||||||
def test_build_prompt_tokens_no_image_for_text_only_tasks():
|
def test_build_prompt_tokens_no_image_for_text_only_tasks():
|
||||||
tok = FakeTokenizer()
|
tok = FakeTokenizer()
|
||||||
ids = build_prompt_tokens("hi", tok, task="t2t")
|
result = build_prompt_tokens("hi", tok, task="t2t")
|
||||||
|
ids = result.token_ids
|
||||||
assert FakeTokenizer.SPECIAL["<img>"] not in ids, "<img> must NOT appear for text-only tasks"
|
assert FakeTokenizer.SPECIAL["<img>"] not in ids, "<img> must NOT appear for text-only tasks"
|
||||||
|
|
||||||
|
|
||||||
@@ -203,14 +205,16 @@ def test_build_prompt_tokens_no_image_for_text_only_tasks():
|
|||||||
def test_build_prompt_tokens_trigger_is_last_token(task: str, trigger_id: int):
|
def test_build_prompt_tokens_trigger_is_last_token(task: str, trigger_id: int):
|
||||||
"""Trigger tag id must be the LAST token (after `Assistant: ` segment)."""
|
"""Trigger tag id must be the LAST token (after `Assistant: ` segment)."""
|
||||||
tok = FakeTokenizer()
|
tok = FakeTokenizer()
|
||||||
ids = build_prompt_tokens("hi", tok, task=task)
|
result = build_prompt_tokens("hi", tok, task=task)
|
||||||
|
ids = result.token_ids
|
||||||
assert ids[-1] == trigger_id
|
assert ids[-1] == trigger_id
|
||||||
|
|
||||||
|
|
||||||
def test_build_prompt_tokens_no_trigger_for_plain_tasks():
|
def test_build_prompt_tokens_no_trigger_for_plain_tasks():
|
||||||
"""Tasks without trigger_tag (t2t / i2t) must NOT append a trigger id."""
|
"""Tasks without trigger_tag (t2t / i2t) must NOT append a trigger id."""
|
||||||
tok = FakeTokenizer()
|
tok = FakeTokenizer()
|
||||||
ids = build_prompt_tokens("hi", tok, task="t2t")
|
result = build_prompt_tokens("hi", tok, task="t2t")
|
||||||
|
ids = result.token_ids
|
||||||
assert ids[-1] not in {
|
assert ids[-1] not in {
|
||||||
FakeTokenizer.SPECIAL["<think>"],
|
FakeTokenizer.SPECIAL["<think>"],
|
||||||
FakeTokenizer.SPECIAL["<recaption>"],
|
FakeTokenizer.SPECIAL["<recaption>"],
|
||||||
@@ -305,7 +309,8 @@ def test_segment_tokenize_diverges_from_full_string_encode():
|
|||||||
tok = AutoTokenizer.from_pretrained(_HUNYUAN_MODEL_ID, trust_remote_code=True)
|
tok = AutoTokenizer.from_pretrained(_HUNYUAN_MODEL_ID, trust_remote_code=True)
|
||||||
|
|
||||||
user_prompt = "写一首关于夜的诗。"
|
user_prompt = "写一首关于夜的诗。"
|
||||||
seg_ids = build_prompt_tokens(user_prompt, tok, task="i2t")
|
result = build_prompt_tokens(user_prompt, tok, task="i2t")
|
||||||
|
seg_ids = result.token_ids
|
||||||
full_ids = tok.encode(build_prompt(user_prompt, task="i2t"), add_special_tokens=False)
|
full_ids = tok.encode(build_prompt(user_prompt, task="i2t"), add_special_tokens=False)
|
||||||
|
|
||||||
assert seg_ids != full_ids, (
|
assert seg_ids != full_ids, (
|
||||||
|
|||||||
@@ -93,7 +93,9 @@ def _run(stage_config_path: str, output_path: Path) -> tuple[Image.Image, str, f
|
|||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||||
token_ids = build_prompt_tokens(PROMPT, tokenizer, task="it2i_recaption", sys_type="en_unified")
|
result = build_prompt_tokens(PROMPT, tokenizer, task="it2i_recaption", sys_type="en_unified")
|
||||||
|
token_ids = result.token_ids
|
||||||
|
system_prompt_type = result.system_prompt_type
|
||||||
|
|
||||||
with OmniRunner(MODEL_NAME, stage_configs_path=stage_config_path) as runner:
|
with OmniRunner(MODEL_NAME, stage_configs_path=stage_config_path) as runner:
|
||||||
params_list = list(runner.omni.default_sampling_params_list)
|
params_list = list(runner.omni.default_sampling_params_list)
|
||||||
@@ -108,7 +110,7 @@ def _run(stage_config_path: str, output_path: Path) -> tuple[Image.Image, str, f
|
|||||||
{
|
{
|
||||||
"prompt_token_ids": token_ids,
|
"prompt_token_ids": token_ids,
|
||||||
"prompt": PROMPT,
|
"prompt": PROMPT,
|
||||||
"use_system_prompt": "en_unified",
|
"use_system_prompt": system_prompt_type,
|
||||||
"height": HEIGHT,
|
"height": HEIGHT,
|
||||||
"width": WIDTH,
|
"width": WIDTH,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ canonical mapping for both flows.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .system_prompt import get_system_prompt
|
from .system_prompt import get_system_prompt
|
||||||
@@ -128,13 +129,19 @@ def build_prompt(
|
|||||||
return "".join(parts)
|
return "".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PromptTokensResult:
|
||||||
|
token_ids: list[int] # The tokenized prompt
|
||||||
|
system_prompt_type: str # The effective system prompt type used
|
||||||
|
|
||||||
|
|
||||||
def build_prompt_tokens(
|
def build_prompt_tokens(
|
||||||
user_prompt: str,
|
user_prompt: str,
|
||||||
tokenizer,
|
tokenizer,
|
||||||
task: str = "it2i_think",
|
task: str = "it2i_think",
|
||||||
sys_type: str | None = None,
|
sys_type: str | None = None,
|
||||||
custom_system_prompt: str | None = None,
|
custom_system_prompt: str | None = None,
|
||||||
) -> list[int]:
|
) -> PromptTokensResult:
|
||||||
"""Segment-by-segment tokenization that matches HF apply_chat_template.
|
"""Segment-by-segment tokenization that matches HF apply_chat_template.
|
||||||
|
|
||||||
Calling tokenizer.encode(build_prompt(...)) on the full string lets BPE
|
Calling tokenizer.encode(build_prompt(...)) on the full string lets BPE
|
||||||
@@ -144,6 +151,9 @@ def build_prompt_tokens(
|
|||||||
each segment independently and concatenates token_ids, so no cross-
|
each segment independently and concatenates token_ids, so no cross-
|
||||||
boundary merge happens. We replicate that here and feed the result to
|
boundary merge happens. We replicate that here and feed the result to
|
||||||
Omni via OmniTokensPrompt (prompt_token_ids).
|
Omni via OmniTokensPrompt (prompt_token_ids).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
PromptTokensResult
|
||||||
"""
|
"""
|
||||||
if task not in _TASK_PRESETS:
|
if task not in _TASK_PRESETS:
|
||||||
raise ValueError(f"Unknown task {task!r}. Choose from: {available_tasks()}")
|
raise ValueError(f"Unknown task {task!r}. Choose from: {available_tasks()}")
|
||||||
@@ -162,7 +172,11 @@ def build_prompt_tokens(
|
|||||||
# protect, fall back to whole-string encode.
|
# protect, fall back to whole-string encode.
|
||||||
if task == "t2i_vanilla":
|
if task == "t2i_vanilla":
|
||||||
s = build_prompt(user_prompt, task, sys_type, custom_system_prompt)
|
s = build_prompt(user_prompt, task, sys_type, custom_system_prompt)
|
||||||
return tokenizer.encode(s, add_special_tokens=False)
|
token_ids = tokenizer.encode(s, add_special_tokens=False)
|
||||||
|
return PromptTokensResult(
|
||||||
|
token_ids=token_ids,
|
||||||
|
system_prompt_type=effective_sys_type,
|
||||||
|
)
|
||||||
|
|
||||||
system_prompt = get_system_prompt(effective_sys_type, preset_bot_task, custom_system_prompt)
|
system_prompt = get_system_prompt(effective_sys_type, preset_bot_task, custom_system_prompt)
|
||||||
# Do NOT strip -- HF apply_chat_template keeps the system prompt's
|
# Do NOT strip -- HF apply_chat_template keeps the system prompt's
|
||||||
@@ -180,7 +194,11 @@ def build_prompt_tokens(
|
|||||||
ids += tokenizer.encode("\n\nAssistant: ", add_special_tokens=False)
|
ids += tokenizer.encode("\n\nAssistant: ", add_special_tokens=False)
|
||||||
if trig_id is not None:
|
if trig_id is not None:
|
||||||
ids += [trig_id]
|
ids += [trig_id]
|
||||||
return ids
|
|
||||||
|
return PromptTokensResult(
|
||||||
|
token_ids=ids,
|
||||||
|
system_prompt_type=effective_sys_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["build_prompt", "build_prompt_tokens", "resolve_stop_token_ids", _TASK_PRESETS]
|
__all__ = ["build_prompt", "build_prompt_tokens", "resolve_stop_token_ids", _TASK_PRESETS]
|
||||||
@@ -2255,26 +2255,32 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
|||||||
else:
|
else:
|
||||||
engine_prompt_data = {"image": reference_images}
|
engine_prompt_data = {"image": reference_images}
|
||||||
|
|
||||||
prompt_token_ids: list[int] | None = None
|
engine_prompt: OmniTextPrompt = {"prompt": prompt}
|
||||||
if bot_task:
|
if bot_task:
|
||||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||||
build_prompt,
|
build_prompt,
|
||||||
build_prompt_tokens,
|
build_prompt_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
prompt_token_ids: list[int] | None = None
|
||||||
|
system_prompt_type: str | None = None
|
||||||
if tokenizer is not None:
|
if tokenizer is not None:
|
||||||
prompt_token_ids = build_prompt_tokens(prompt, tokenizer, task=bot_task)
|
result = build_prompt_tokens(prompt, tokenizer, task=bot_task)
|
||||||
|
prompt_token_ids = result.token_ids
|
||||||
|
system_prompt_type = result.system_prompt_type
|
||||||
else:
|
else:
|
||||||
prompt = build_prompt(prompt, task=bot_task)
|
prompt = build_prompt(prompt, task=bot_task)
|
||||||
|
engine_prompt["prompt"] = prompt
|
||||||
|
|
||||||
if reference_images and len(reference_images) == 1:
|
if reference_images and len(reference_images) == 1:
|
||||||
engine_prompt_data = {"image": reference_images[0]}
|
engine_prompt_data = {"image": reference_images[0]}
|
||||||
modalities = ["image"]
|
modalities = ["image"]
|
||||||
|
if prompt_token_ids is not None:
|
||||||
|
engine_prompt["prompt_token_ids"] = prompt_token_ids
|
||||||
|
if system_prompt_type is not None:
|
||||||
|
engine_prompt["use_system_prompt"] = system_prompt_type
|
||||||
|
|
||||||
engine_prompt: OmniTextPrompt = {"prompt": prompt}
|
|
||||||
engine_prompt["modalities"] = modalities
|
engine_prompt["modalities"] = modalities
|
||||||
if prompt_token_ids is not None:
|
|
||||||
engine_prompt["prompt_token_ids"] = prompt_token_ids
|
|
||||||
if negative_prompt is not None:
|
if negative_prompt is not None:
|
||||||
engine_prompt["negative_prompt"] = negative_prompt
|
engine_prompt["negative_prompt"] = negative_prompt
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user