[Bugfix] Route text-only chat as per-request comprehension in HunyuanImage3 AR sampler (#6111)

Signed-off-by: MrlixiangWE <mrdanaer@gmail.com>
Co-authored-by: zijianc2 <157244773+zijianc2@users.noreply.github.com>
This commit is contained in:
DanaerLeeandzijianc2 authored and GitHub committed 2026-09-20 15:23:53 +00:00
1 parent 102ba716bf
commit 16734bbdf3
9 files changed
+485 -31

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for HunyuanImage3 AR sampler logic (stage transitions,
ratio restriction, comprehension blocking)."""
@@ -21,6 +21,10 @@ RATIO_END = 210
RATIO_OTHER_START = 220
RATIO_OTHER_END = 223
TIMESTEP = 224
CFG = 110
IMG_TOK = 111
JOINT_SEP = 112
EOI = 113
class FakeSamplerModel:
@@ -47,10 +51,13 @@ class FakeSamplerModel:
self._stage_transitions[END_OF_THINK] = [RECAPTION]
self._stage_transitions[END_OF_RECAPTION] = [ANSWER, BOI, SIZE_TOKEN]
self._blocked_token_ids: set[int] = set()
if is_comprehension:
self._blocked_token_ids.update([BOI, SIZE_TOKEN])
self._blocked_token_ids.update(self._all_ratio_ids)
# Mirrors the model: the blocklist is always built so generation-mode
# engines can apply it to per-request comprehension rows (#6088).
self._cfg_token_id = CFG
self._blocked_token_ids: set[int] = {BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP}
self._blocked_token_ids.update(self._all_ratio_ids)
self._blocked_token_ids_tensor = None
self._sampler = None
# Bind the real methods from the model class.
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
@@ -59,6 +66,8 @@ class FakeSamplerModel:
_get_forced_token = _Real._get_forced_token
_apply_ratio_restriction = _Real._apply_ratio_restriction
_row_is_comprehension = staticmethod(_Real._row_is_comprehension)
sample = _Real.sample
class FakeTokenEmbedding:
@@ -250,3 +259,114 @@ class TestForceEosAfterRatio:
assert logits[0, EOS].item() == 0
non_eos_max = logits[0, :EOS].max().item()
assert non_eos_max == min_score
class _RecordingSampler:
"""Stub sampler: returns the (mutated) logits so tests can assert on them."""
def __call__(self, *, logits, sampling_metadata):
return logits
def _metadata(output_token_ids):
from types import SimpleNamespace
return SimpleNamespace(output_token_ids=output_token_ids)
class TestPerRequestTaskMode:
"""Generation-mode engines must honor per-request comprehension rows (#6088).
A text chat (I2T/T2T) served by an AR+DiT deployment is marked with
extra_args={"ar_task_mode": "comprehension"}; its rows get the stateless
image-token blocking instead of stage-transition forcing, while untagged
rows keep the generation behavior.
"""
VOCAB = 300
def _run(self, model, histories, per_req_extra_args):
logits = torch.zeros(len(histories), self.VOCAB)
logits[:, 50] = 7.0 # ordinary text token, must survive comprehension
model._sampler = _RecordingSampler()
out = model.sample(logits, _metadata(histories), per_req_extra_args=per_req_extra_args)
return out
def test_comprehension_row_skips_forcing_and_blocks_image_tokens(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}, None],
)
# Row 0 (comprehension): no </think> -> <recaption> forcing (ordinary
# text token survives, which a force-fill would have erased) and every
# DiT scaffold token is masked.
assert logits[0, 50].item() == 7.0
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP, RATIO_START, RATIO_OTHER_START):
assert logits[0, tid].item() == min_score, f"token {tid} not blocked"
# Row 1 (generation): forced to <recaption> exactly as before.
assert logits[1, RECAPTION].item() == 0
assert logits[1, 50].item() == min_score
def test_no_extra_args_keeps_generation_behavior(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(model, histories=[[END_OF_THINK]], per_req_extra_args=None)
assert logits[0, RECAPTION].item() == 0
assert logits[0, 50].item() == min_score
def test_empty_or_foreign_extra_args_keeps_generation_behavior(self):
model = FakeSamplerModel(is_comprehension=False)
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{}, {"other_key": 1}],
)
assert logits[0, RECAPTION].item() == 0
assert logits[1, RECAPTION].item() == 0
def test_comprehension_row_without_trigger_history(self):
model = FakeSamplerModel(is_comprehension=False)
min_score = torch.finfo(torch.float32).min
logits = self._run(
model,
histories=[[1, 2, 3]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
)
assert logits[0, 50].item() == 7.0
assert logits[0, CFG].item() == min_score
def test_static_comprehension_engine_blocks_cfg(self):
model = FakeSamplerModel(is_comprehension=True)
min_score = torch.finfo(torch.float32).min
logits = self._run(model, histories=[[1, 2]], per_req_extra_args=None)
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP):
assert logits[0, tid].item() == min_score
assert logits[0, 50].item() == 7.0
def test_extra_args_shorter_than_batch_defaults_to_generation(self):
model = FakeSamplerModel(is_comprehension=False)
logits = self._run(
model,
histories=[[END_OF_THINK], [END_OF_THINK]],
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
)
min_score = torch.finfo(torch.float32).min
assert logits[0, 50].item() == 7.0 # tagged row: comprehension
assert logits[1, RECAPTION].item() == 0 # missing entry: generation
assert logits[1, 50].item() == min_score
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Unit tests for OmniOpenAIServingChat sampling params handling.
@@ -8,6 +9,7 @@ are correctly applied to the comprehension stage while preserving YAML defaults.
import asyncio
from types import SimpleNamespace
from typing import Any, cast
import pytest
from pytest_mock import MockerFixture
@@ -326,6 +328,64 @@ def test_mixed_consumer_keeps_root_common_args_with_nested_extras(mock_engine_cl
assert captured["prompt"]["negative_prompt"] == "avoid blur"
def test_text_only_request_reaches_engine_with_comprehension_task_mode(mock_engine_client, mocker: MockerFixture):
"""Exercise the production chat path, not only the tagging helper."""
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
mock_engine_client.stage_configs = [
SimpleNamespace(stage_type="llm", is_comprehension=True),
SimpleNamespace(stage_type="diffusion", is_comprehension=False),
]
mock_engine_client.default_sampling_params_list = [
SamplingParams(),
OmniDiffusionSamplingParams(),
]
mock_engine_client.output_modalities = ["text", "image"]
mock_engine_client.errored = False
mock_engine_client.renderer = SimpleNamespace(get_tokenizer=lambda: object())
captured: dict[str, object] = {}
async def results():
if False:
yield None
def generate(**kwargs):
captured.update(kwargs)
return results()
mock_engine_client.generate = generate
serving_chat = build_serving_chat(
engine_client=mock_engine_client,
models=SimpleNamespace(model_name=lambda _: "test"),
online_renderer=SimpleNamespace(validate_chat_template=lambda **_: None),
trust_request_chat_template=True,
)
serving_chat._diffusion_mode = False
serving_chat._diffusion_extra_body_params = frozenset()
mocker.patch.multiple(
serving_chat,
_check_model=mocker.AsyncMock(return_value=None),
_maybe_get_adapters=mocker.Mock(return_value=None),
_effective_chat_template_kwargs=mocker.Mock(return_value={}),
_preprocess_chat=mocker.AsyncMock(return_value=([], [{"prompt": "raw"}])),
_base_request_id=mocker.Mock(return_value="test"),
_log_inputs=mocker.Mock(),
chat_completion_full_generator=mocker.AsyncMock(return_value="done"),
)
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "describe this image"}],
modalities=["text"],
)
assert asyncio.run(serving_chat._create_chat_completion(request)) == "done"
sampling_params_list = cast(list[Any], captured["sampling_params_list"])
assert sampling_params_list[0].extra_args == {"ar_task_mode": "comprehension"}
assert sampling_params_list[1].extra_args == {}
assert captured["output_modalities"] == ["text"]
@pytest.fixture
def mock_request(mocker: MockerFixture):
"""Create a mock request with all OpenAI sampling params set to None."""
@@ -865,3 +925,54 @@ class TestResolveHeightWidth:
h, w = OmniOpenAIServingChat._resolve_height_width_from_extra_body({"size": "invalid"})
assert h is None
assert w is None
# Tests for _apply_text_chat_ar_task_mode (#6088)
def _tag_request(modalities):
return SimpleNamespace(modalities=modalities)
def _apply_tag(params, modalities):
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
OmniOpenAIServingChat._apply_text_chat_ar_task_mode(params, _tag_request(modalities))
return params
@pytest.mark.parametrize("modalities", [[], ["text"]])
def test_text_only_chat_tags_ar_stage_as_comprehension(modalities):
ar = SamplingParams()
params = _apply_tag([ar], modalities)
assert params[0].extra_args == {"ar_task_mode": "comprehension"}
@pytest.mark.parametrize("modalities", [["image"], ["text", "audio"], ["video"]])
def test_non_text_output_request_is_untouched(modalities):
ar = SamplingParams()
_apply_tag([ar], modalities)
assert ar.extra_args is None
def test_diffusion_stage_params_are_untouched():
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
ar = SamplingParams()
dit = OmniDiffusionSamplingParams()
dit_extra_before = getattr(dit, "extra_args", None)
_apply_tag([ar, dit], ["text"])
assert ar.extra_args == {"ar_task_mode": "comprehension"}
assert getattr(dit, "extra_args", None) == dit_extra_before
def test_explicit_caller_ar_task_mode_is_preserved():
ar = SamplingParams(extra_args={"ar_task_mode": "generation"})
_apply_tag([ar], ["text"])
assert ar.extra_args["ar_task_mode"] == "generation"
def test_existing_extra_args_are_merged_not_replaced():
ar = SamplingParams(extra_args={"custom": 1})
_apply_tag([ar], None)
assert ar.extra_args == {"custom": 1, "ar_task_mode": "comprehension"}
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for per-request model sampler arguments.
Custom model samplers that set ``model_sampler_wants_extra_args = True``
(e.g. HunyuanImage3's per-request ``ar_task_mode``, #6088) receive one
``SamplingParams.extra_args`` entry per batch row, in ``input_batch.req_ids``
order — the same order ``_build_model_sampler_output_token_ids`` uses. Pure CPU.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.worker.sampling_utils import build_model_sampler_extra_args, call_model_sampler
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _state(extra_args):
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra_args))
def test_row_order_follows_req_ids():
input_batch = SimpleNamespace(req_ids=["b", "a"])
requests = {
"a": _state({"ar_task_mode": "comprehension"}),
"b": _state(None),
}
assert build_model_sampler_extra_args(input_batch, requests) == [None, {"ar_task_mode": "comprehension"}]
def test_missing_request_state_yields_none():
assert build_model_sampler_extra_args(SimpleNamespace(req_ids=["gone"]), {}) == [None]
def test_request_without_sampling_params_yields_none():
input_batch = SimpleNamespace(req_ids=["r0"])
assert build_model_sampler_extra_args(input_batch, {"r0": SimpleNamespace(sampling_params=None)}) == [None]
def test_empty_batch():
assert build_model_sampler_extra_args(SimpleNamespace(req_ids=[]), {}) == []
def test_opted_in_sampler_receives_per_request_extra_args(mocker):
model = SimpleNamespace(model_sampler_wants_extra_args=True)
model_sample = mocker.Mock(return_value="sampled")
input_batch = SimpleNamespace(req_ids=["a", "b"])
logits = torch.empty(0)
metadata = SimpleNamespace()
requests = {
"a": _state(None),
"b": _state({"ar_task_mode": "comprehension"}),
}
result = call_model_sampler(
model,
model_sample,
logits,
metadata,
input_batch=input_batch,
requests=requests,
)
assert result == "sampled"
model_sample.assert_called_once_with(
logits,
metadata,
per_req_extra_args=[None, {"ar_task_mode": "comprehension"}],
)
def test_non_opted_in_sampler_keeps_two_argument_contract(mocker):
model = SimpleNamespace(model_sampler_wants_extra_args=False)
model_sample = mocker.Mock(return_value="sampled")
logits = torch.empty(0)
metadata = SimpleNamespace()
result = call_model_sampler(
model,
model_sample,
logits,
metadata,
input_batch=SimpleNamespace(req_ids=["a"]),
requests={"a": _state({"ar_task_mode": "comprehension"})},
)
assert result == "sampled"
model_sample.assert_called_once_with(logits, metadata)
+1 -1
View File
@@ -2639,7 +2639,7 @@ _REQUIRED_DUPLEX_CALLS = {
"_sample": "_apply_duplex_sampling",
}
_MODEL_SAMPLER_CALL = "model_sample"
_MODEL_SAMPLER_CALL = "call_model_sampler"
def _repo_root():