[Bugfix] Route text-only chat as per-request comprehension in HunyuanImage3 AR sampler (#6111)

Signed-off-by: MrlixiangWE <mrdanaer@gmail.com>
Co-authored-by: zijianc2 <157244773+zijianc2@users.noreply.github.com>
This commit is contained in:
DanaerLeeandzijianc2 authored and GitHub committed 2026-09-20 15:23:53 +00:00
1 parent 102ba716bf
commit 16734bbdf3
9 files changed
+485 -31

No files matched your search

@@ -1448,6 +1448,9 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
prefer_model_sampler = True
logitsprocs_need_output_token_ids = True
# Ask the AR model runner to pass each row's SamplingParams.extra_args to
# sample(), so per-request task modes (ar_task_mode) can reach the sampler.
model_sampler_wants_extra_args = True
# Siglip2 ViT supports data-parallel encoding (mm_encoder_tp_mode="data"):
# weights are replicated and the image batch is sharded across TP ranks.
@@ -1571,24 +1574,37 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self._all_ratio_ids.update(range(s, e))
# Determine mode: comprehension (I2T/T2T) vs generation (IT2I/T2I).
# ``engine_output_type`` is a per-engine static (from the pipeline
# registry), but a generation deployment (e.g. the AR+DiT
# hunyuan_image_3_moe pipeline, engine_output_type="latent") also
# serves plain chat completions whose answer is text (I2T/T2T).
# Those requests are marked per-request via
# ``SamplingParams.extra_args["ar_task_mode"] == "comprehension"``
# by the serving layer and must not go through the image-generation
# stage transitions below (#6088).
engine_output_type = getattr(vllm_config.model_config, "engine_output_type", None)
self._is_comprehension = engine_output_type in (None, "text")
# For comprehension mode, block image generation tokens but allow
# For comprehension requests, block image generation tokens but allow
# text structure tokens (<think>, <answer>, etc.) so the model can
# follow its natural generation pattern. Runtime sampling params
# decide stop tokens from the active bot_task, matching the official
# HunyuanImage3 generation path without hard-coded YAML token ids.
self._blocked_token_ids: set[int] = set()
if self._is_comprehension:
self._blocked_token_ids.update(
[
self._mrope_boi_token_id, # <boi>
self._mrope_eoi_token_id, # <eoi>
self._size_token_id, # <img_size_*>
]
)
self._blocked_token_ids.update(self._all_ratio_ids)
# <cfg>, <img>, <timestep> and <joint_img_sep> are DiT-side scaffold
# tokens that are never valid in a text answer; without them in the
# blocklist an I2T request can degenerate into a stream of <cfg>
# tokens (#6088).
self._cfg_token_id = tokenizer.convert_tokens_to_ids("<cfg>")
self._blocked_token_ids: set[int] = {
self._mrope_boi_token_id, # <boi>
self._mrope_eoi_token_id, # <eoi>
self._size_token_id, # <img_size_*>
self._cfg_token_id, # <cfg>
self._mrope_img_token_id, # <img>
self._timestep_token_id, # <timestep>
self._mrope_joint_img_sep_token_id, # <joint_img_sep>
}
self._blocked_token_ids.update(self._all_ratio_ids)
# For generation mode, build stage transition map.
# Official logic: </think> → [<recaption>],
@@ -2061,6 +2077,7 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
per_req_extra_args: Sequence[Mapping[str, Any] | None] | None = None,
) -> SamplerOutput | None:
if logits is None or logits.numel() == 0:
return None
@@ -2070,25 +2087,34 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
min_score = torch.finfo(logits.dtype).min
if self._blocked_token_ids_tensor is None and self._blocked_token_ids:
self._blocked_token_ids_tensor = torch.tensor(
sorted(self._blocked_token_ids),
dtype=torch.long,
device=logits.device,
)
if self._is_comprehension:
# Comprehension path is stateless: we only need to mask a fixed
# set of blocked token ids on every step. Do it in one batched
# index_fill_ instead of a per-(req, id) Python loop of scalar
# GPU writes, and skip reading `output_token_ids` from CPU
# (unused here) so this branch never forces a D2H sync.
if self._blocked_token_ids_tensor is None and self._blocked_token_ids:
self._blocked_token_ids_tensor = torch.tensor(
sorted(self._blocked_token_ids),
dtype=torch.long,
device=logits.device,
)
if self._blocked_token_ids_tensor is not None:
logits.index_fill_(-1, self._blocked_token_ids_tensor, min_score)
return self._sampler(logits=logits, sampling_metadata=sampling_metadata)
# Generation path retains the per-request stateful logic for forced
# stage-transition tokens and ratio-restriction.
# Generation-mode engine. Individual requests can still be
# comprehension (I2T/T2T chat served by an AR+DiT deployment); the
# serving layer marks those with extra_args["ar_task_mode"] ==
# "comprehension", and they get the stateless blocking above instead
# of stage-transition forcing (#6088).
for req_idx in range(logits.shape[0]):
if self._row_is_comprehension(per_req_extra_args, req_idx):
if self._blocked_token_ids_tensor is not None:
logits[req_idx].index_fill_(-1, self._blocked_token_ids_tensor, min_score)
continue
decoded_tokens: list[int] = (
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
)
@@ -2106,6 +2132,24 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
return self._sampler(logits=logits, sampling_metadata=sampling_metadata)
@staticmethod
def _row_is_comprehension(
per_req_extra_args: Sequence[Mapping[str, Any] | None] | None,
req_idx: int,
) -> bool:
"""Whether this batch row is a per-request comprehension override.
Defaults to False (generation) when no extra_args reached the
sampler, so offline ``Omni.generate`` image-generation flows keep
the stage-transition behavior unchanged.
"""
if per_req_extra_args is None or req_idx >= len(per_req_extra_args):
return False
extra_args = per_req_extra_args[req_idx]
if not extra_args:
return False
return extra_args.get("ar_task_mode") == "comprehension"
def _get_forced_token(self, decoded_tokens: list[int]) -> int | None:
"""Derive the next forced token from output history (stateless).