[Bugfix] Route text-only chat as per-request comprehension in HunyuanImage3 AR sampler (#6111)
Signed-off-by: MrlixiangWE <mrdanaer@gmail.com> Co-authored-by: zijianc2 <157244773+zijianc2@users.noreply.github.com>
This commit is contained in:
1 parent
102ba716bf
commit
16734bbdf3
9 files changed
+485
-31
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for HunyuanImage3 AR sampler logic (stage transitions,
|
||||
ratio restriction, comprehension blocking)."""
|
||||
|
||||
@@ -21,6 +21,10 @@ RATIO_END = 210
|
||||
RATIO_OTHER_START = 220
|
||||
RATIO_OTHER_END = 223
|
||||
TIMESTEP = 224
|
||||
CFG = 110
|
||||
IMG_TOK = 111
|
||||
JOINT_SEP = 112
|
||||
EOI = 113
|
||||
|
||||
|
||||
class FakeSamplerModel:
|
||||
@@ -47,10 +51,13 @@ class FakeSamplerModel:
|
||||
self._stage_transitions[END_OF_THINK] = [RECAPTION]
|
||||
self._stage_transitions[END_OF_RECAPTION] = [ANSWER, BOI, SIZE_TOKEN]
|
||||
|
||||
self._blocked_token_ids: set[int] = set()
|
||||
if is_comprehension:
|
||||
self._blocked_token_ids.update([BOI, SIZE_TOKEN])
|
||||
self._blocked_token_ids.update(self._all_ratio_ids)
|
||||
# Mirrors the model: the blocklist is always built so generation-mode
|
||||
# engines can apply it to per-request comprehension rows (#6088).
|
||||
self._cfg_token_id = CFG
|
||||
self._blocked_token_ids: set[int] = {BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP}
|
||||
self._blocked_token_ids.update(self._all_ratio_ids)
|
||||
self._blocked_token_ids_tensor = None
|
||||
self._sampler = None
|
||||
|
||||
# Bind the real methods from the model class.
|
||||
from vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3 import (
|
||||
@@ -59,6 +66,8 @@ class FakeSamplerModel:
|
||||
|
||||
_get_forced_token = _Real._get_forced_token
|
||||
_apply_ratio_restriction = _Real._apply_ratio_restriction
|
||||
_row_is_comprehension = staticmethod(_Real._row_is_comprehension)
|
||||
sample = _Real.sample
|
||||
|
||||
|
||||
class FakeTokenEmbedding:
|
||||
@@ -250,3 +259,114 @@ class TestForceEosAfterRatio:
|
||||
assert logits[0, EOS].item() == 0
|
||||
non_eos_max = logits[0, :EOS].max().item()
|
||||
assert non_eos_max == min_score
|
||||
|
||||
|
||||
class _RecordingSampler:
|
||||
"""Stub sampler: returns the (mutated) logits so tests can assert on them."""
|
||||
|
||||
def __call__(self, *, logits, sampling_metadata):
|
||||
return logits
|
||||
|
||||
|
||||
def _metadata(output_token_ids):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(output_token_ids=output_token_ids)
|
||||
|
||||
|
||||
class TestPerRequestTaskMode:
|
||||
"""Generation-mode engines must honor per-request comprehension rows (#6088).
|
||||
|
||||
A text chat (I2T/T2T) served by an AR+DiT deployment is marked with
|
||||
extra_args={"ar_task_mode": "comprehension"}; its rows get the stateless
|
||||
image-token blocking instead of stage-transition forcing, while untagged
|
||||
rows keep the generation behavior.
|
||||
"""
|
||||
|
||||
VOCAB = 300
|
||||
|
||||
def _run(self, model, histories, per_req_extra_args):
|
||||
logits = torch.zeros(len(histories), self.VOCAB)
|
||||
logits[:, 50] = 7.0 # ordinary text token, must survive comprehension
|
||||
model._sampler = _RecordingSampler()
|
||||
out = model.sample(logits, _metadata(histories), per_req_extra_args=per_req_extra_args)
|
||||
return out
|
||||
|
||||
def test_comprehension_row_skips_forcing_and_blocks_image_tokens(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}, None],
|
||||
)
|
||||
|
||||
# Row 0 (comprehension): no </think> -> <recaption> forcing (ordinary
|
||||
# text token survives, which a force-fill would have erased) and every
|
||||
# DiT scaffold token is masked.
|
||||
assert logits[0, 50].item() == 7.0
|
||||
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP, RATIO_START, RATIO_OTHER_START):
|
||||
assert logits[0, tid].item() == min_score, f"token {tid} not blocked"
|
||||
|
||||
# Row 1 (generation): forced to <recaption> exactly as before.
|
||||
assert logits[1, RECAPTION].item() == 0
|
||||
assert logits[1, 50].item() == min_score
|
||||
|
||||
def test_no_extra_args_keeps_generation_behavior(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(model, histories=[[END_OF_THINK]], per_req_extra_args=None)
|
||||
|
||||
assert logits[0, RECAPTION].item() == 0
|
||||
assert logits[0, 50].item() == min_score
|
||||
|
||||
def test_empty_or_foreign_extra_args_keeps_generation_behavior(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{}, {"other_key": 1}],
|
||||
)
|
||||
|
||||
assert logits[0, RECAPTION].item() == 0
|
||||
assert logits[1, RECAPTION].item() == 0
|
||||
|
||||
def test_comprehension_row_without_trigger_history(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[1, 2, 3]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
|
||||
)
|
||||
|
||||
assert logits[0, 50].item() == 7.0
|
||||
assert logits[0, CFG].item() == min_score
|
||||
|
||||
def test_static_comprehension_engine_blocks_cfg(self):
|
||||
model = FakeSamplerModel(is_comprehension=True)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
logits = self._run(model, histories=[[1, 2]], per_req_extra_args=None)
|
||||
|
||||
for tid in (BOI, EOI, SIZE_TOKEN, CFG, IMG_TOK, TIMESTEP, JOINT_SEP):
|
||||
assert logits[0, tid].item() == min_score
|
||||
assert logits[0, 50].item() == 7.0
|
||||
|
||||
def test_extra_args_shorter_than_batch_defaults_to_generation(self):
|
||||
model = FakeSamplerModel(is_comprehension=False)
|
||||
|
||||
logits = self._run(
|
||||
model,
|
||||
histories=[[END_OF_THINK], [END_OF_THINK]],
|
||||
per_req_extra_args=[{"ar_task_mode": "comprehension"}],
|
||||
)
|
||||
min_score = torch.finfo(torch.float32).min
|
||||
|
||||
assert logits[0, 50].item() == 7.0 # tagged row: comprehension
|
||||
assert logits[1, RECAPTION].item() == 0 # missing entry: generation
|
||||
assert logits[1, 50].item() == min_score
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
Unit tests for OmniOpenAIServingChat sampling params handling.
|
||||
|
||||
@@ -8,6 +9,7 @@ are correctly applied to the comprehension stage while preserving YAML defaults.
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -326,6 +328,64 @@ def test_mixed_consumer_keeps_root_common_args_with_nested_extras(mock_engine_cl
|
||||
assert captured["prompt"]["negative_prompt"] == "avoid blur"
|
||||
|
||||
|
||||
def test_text_only_request_reaches_engine_with_comprehension_task_mode(mock_engine_client, mocker: MockerFixture):
|
||||
"""Exercise the production chat path, not only the tagging helper."""
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(stage_type="llm", is_comprehension=True),
|
||||
SimpleNamespace(stage_type="diffusion", is_comprehension=False),
|
||||
]
|
||||
mock_engine_client.default_sampling_params_list = [
|
||||
SamplingParams(),
|
||||
OmniDiffusionSamplingParams(),
|
||||
]
|
||||
mock_engine_client.output_modalities = ["text", "image"]
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.renderer = SimpleNamespace(get_tokenizer=lambda: object())
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def results():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
def generate(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return results()
|
||||
|
||||
mock_engine_client.generate = generate
|
||||
serving_chat = build_serving_chat(
|
||||
engine_client=mock_engine_client,
|
||||
models=SimpleNamespace(model_name=lambda _: "test"),
|
||||
online_renderer=SimpleNamespace(validate_chat_template=lambda **_: None),
|
||||
trust_request_chat_template=True,
|
||||
)
|
||||
serving_chat._diffusion_mode = False
|
||||
serving_chat._diffusion_extra_body_params = frozenset()
|
||||
mocker.patch.multiple(
|
||||
serving_chat,
|
||||
_check_model=mocker.AsyncMock(return_value=None),
|
||||
_maybe_get_adapters=mocker.Mock(return_value=None),
|
||||
_effective_chat_template_kwargs=mocker.Mock(return_value={}),
|
||||
_preprocess_chat=mocker.AsyncMock(return_value=([], [{"prompt": "raw"}])),
|
||||
_base_request_id=mocker.Mock(return_value="test"),
|
||||
_log_inputs=mocker.Mock(),
|
||||
chat_completion_full_generator=mocker.AsyncMock(return_value="done"),
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model="test",
|
||||
messages=[{"role": "user", "content": "describe this image"}],
|
||||
modalities=["text"],
|
||||
)
|
||||
|
||||
assert asyncio.run(serving_chat._create_chat_completion(request)) == "done"
|
||||
|
||||
sampling_params_list = cast(list[Any], captured["sampling_params_list"])
|
||||
assert sampling_params_list[0].extra_args == {"ar_task_mode": "comprehension"}
|
||||
assert sampling_params_list[1].extra_args == {}
|
||||
assert captured["output_modalities"] == ["text"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request(mocker: MockerFixture):
|
||||
"""Create a mock request with all OpenAI sampling params set to None."""
|
||||
@@ -865,3 +925,54 @@ class TestResolveHeightWidth:
|
||||
h, w = OmniOpenAIServingChat._resolve_height_width_from_extra_body({"size": "invalid"})
|
||||
assert h is None
|
||||
assert w is None
|
||||
|
||||
|
||||
# Tests for _apply_text_chat_ar_task_mode (#6088)
|
||||
|
||||
|
||||
def _tag_request(modalities):
|
||||
return SimpleNamespace(modalities=modalities)
|
||||
|
||||
|
||||
def _apply_tag(params, modalities):
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
OmniOpenAIServingChat._apply_text_chat_ar_task_mode(params, _tag_request(modalities))
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [[], ["text"]])
|
||||
def test_text_only_chat_tags_ar_stage_as_comprehension(modalities):
|
||||
ar = SamplingParams()
|
||||
params = _apply_tag([ar], modalities)
|
||||
assert params[0].extra_args == {"ar_task_mode": "comprehension"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [["image"], ["text", "audio"], ["video"]])
|
||||
def test_non_text_output_request_is_untouched(modalities):
|
||||
ar = SamplingParams()
|
||||
_apply_tag([ar], modalities)
|
||||
assert ar.extra_args is None
|
||||
|
||||
|
||||
def test_diffusion_stage_params_are_untouched():
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
ar = SamplingParams()
|
||||
dit = OmniDiffusionSamplingParams()
|
||||
dit_extra_before = getattr(dit, "extra_args", None)
|
||||
_apply_tag([ar, dit], ["text"])
|
||||
assert ar.extra_args == {"ar_task_mode": "comprehension"}
|
||||
assert getattr(dit, "extra_args", None) == dit_extra_before
|
||||
|
||||
|
||||
def test_explicit_caller_ar_task_mode_is_preserved():
|
||||
ar = SamplingParams(extra_args={"ar_task_mode": "generation"})
|
||||
_apply_tag([ar], ["text"])
|
||||
assert ar.extra_args["ar_task_mode"] == "generation"
|
||||
|
||||
|
||||
def test_existing_extra_args_are_merged_not_replaced():
|
||||
ar = SamplingParams(extra_args={"custom": 1})
|
||||
_apply_tag([ar], None)
|
||||
assert ar.extra_args == {"custom": 1, "ar_task_mode": "comprehension"}
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for per-request model sampler arguments.
|
||||
|
||||
Custom model samplers that set ``model_sampler_wants_extra_args = True``
|
||||
(e.g. HunyuanImage3's per-request ``ar_task_mode``, #6088) receive one
|
||||
``SamplingParams.extra_args`` entry per batch row, in ``input_batch.req_ids``
|
||||
order — the same order ``_build_model_sampler_output_token_ids`` uses. Pure CPU.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.worker.sampling_utils import build_model_sampler_extra_args, call_model_sampler
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _state(extra_args):
|
||||
return SimpleNamespace(sampling_params=SimpleNamespace(extra_args=extra_args))
|
||||
|
||||
|
||||
def test_row_order_follows_req_ids():
|
||||
input_batch = SimpleNamespace(req_ids=["b", "a"])
|
||||
requests = {
|
||||
"a": _state({"ar_task_mode": "comprehension"}),
|
||||
"b": _state(None),
|
||||
}
|
||||
assert build_model_sampler_extra_args(input_batch, requests) == [None, {"ar_task_mode": "comprehension"}]
|
||||
|
||||
|
||||
def test_missing_request_state_yields_none():
|
||||
assert build_model_sampler_extra_args(SimpleNamespace(req_ids=["gone"]), {}) == [None]
|
||||
|
||||
|
||||
def test_request_without_sampling_params_yields_none():
|
||||
input_batch = SimpleNamespace(req_ids=["r0"])
|
||||
assert build_model_sampler_extra_args(input_batch, {"r0": SimpleNamespace(sampling_params=None)}) == [None]
|
||||
|
||||
|
||||
def test_empty_batch():
|
||||
assert build_model_sampler_extra_args(SimpleNamespace(req_ids=[]), {}) == []
|
||||
|
||||
|
||||
def test_opted_in_sampler_receives_per_request_extra_args(mocker):
|
||||
model = SimpleNamespace(model_sampler_wants_extra_args=True)
|
||||
model_sample = mocker.Mock(return_value="sampled")
|
||||
input_batch = SimpleNamespace(req_ids=["a", "b"])
|
||||
logits = torch.empty(0)
|
||||
metadata = SimpleNamespace()
|
||||
requests = {
|
||||
"a": _state(None),
|
||||
"b": _state({"ar_task_mode": "comprehension"}),
|
||||
}
|
||||
|
||||
result = call_model_sampler(
|
||||
model,
|
||||
model_sample,
|
||||
logits,
|
||||
metadata,
|
||||
input_batch=input_batch,
|
||||
requests=requests,
|
||||
)
|
||||
|
||||
assert result == "sampled"
|
||||
model_sample.assert_called_once_with(
|
||||
logits,
|
||||
metadata,
|
||||
per_req_extra_args=[None, {"ar_task_mode": "comprehension"}],
|
||||
)
|
||||
|
||||
|
||||
def test_non_opted_in_sampler_keeps_two_argument_contract(mocker):
|
||||
model = SimpleNamespace(model_sampler_wants_extra_args=False)
|
||||
model_sample = mocker.Mock(return_value="sampled")
|
||||
logits = torch.empty(0)
|
||||
metadata = SimpleNamespace()
|
||||
|
||||
result = call_model_sampler(
|
||||
model,
|
||||
model_sample,
|
||||
logits,
|
||||
metadata,
|
||||
input_batch=SimpleNamespace(req_ids=["a"]),
|
||||
requests={"a": _state({"ar_task_mode": "comprehension"})},
|
||||
)
|
||||
|
||||
assert result == "sampled"
|
||||
model_sample.assert_called_once_with(logits, metadata)
|
||||
@@ -2639,7 +2639,7 @@ _REQUIRED_DUPLEX_CALLS = {
|
||||
"_sample": "_apply_duplex_sampling",
|
||||
}
|
||||
|
||||
_MODEL_SAMPLER_CALL = "model_sample"
|
||||
_MODEL_SAMPLER_CALL = "call_model_sampler"
|
||||
|
||||
|
||||
def _repo_root():
|
||||
|
||||
Reference in new issue
Block a user