fix(glm_image): build the target-shape scaffold without HF's private formatter
Nightly 3034 (Simple · Model Executor Test) failed all six
test_generation_prompt_ids_preserve_hf_target_scaffold cases with
AttributeError: type object 'GlmImageProcessor' has no attribute '_build_prompt_with_target_shape'
The 0.29 rewrite of GlmImageMultiModalProcessor.apply reached for
GlmImageProcessor._build_prompt_with_target_shape, a private transformers
method. The test suite stubs transformers.models.glm_image.processing_glm_image
with a bare class to import glm_image_ar without the real package, so in the
full CPU run the private attribute is not there; the same dependency would
break at runtime on any transformers build that renames or removes it.
Format the scaffold in Omni instead: _build_target_shape_scaffold mirrors the
HF formula (identical in transformers 5.13 and 5.14: target grid, plus the
preview grid for text-to-image, then the image bos token) using only the
processor's public grid_bos_token / grid_eos_token / bos_token attributes.
The test keeps its hard-coded expected suffix, which is what HF produces for
512x768, and no longer imports the HF class.
Validation: tests/model_executor/models/glm_image passes (25 tests) both alone
and inside the full `tests/model_executor -m "core_model and cpu"` run.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MpXe2LeRrUQGjZfKFaocMZ
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
c4b37471c4
commit
121d0b9aee
2 files changed
+34
-12
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for GLM-Image AR model: DataParser, processor, and M-RoPE."""
|
||||
|
||||
import importlib.util
|
||||
@@ -94,18 +94,15 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@pytest.mark.parametrize("num_images", [0, 1, 2])
|
||||
@pytest.mark.parametrize("preformatted", [False, True])
|
||||
def test_generation_prompt_ids_preserve_hf_target_scaffold(num_images, preformatted):
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
from transformers.models.glm_image.processing_glm_image import GlmImageProcessor
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessorInputs
|
||||
|
||||
processor = object.__new__(GlmImageMultiModalProcessor)
|
||||
symbols = SimpleNamespace(grid_bos_token="<sop>", grid_eos_token="<eop>", bos_token="<image_start>")
|
||||
hf_processor = SimpleNamespace(
|
||||
_build_prompt_with_target_shape=partial(GlmImageProcessor._build_prompt_with_target_shape, symbols)
|
||||
)
|
||||
# Only the public token attributes of GlmImageProcessor are needed; the
|
||||
# expected suffix below is what HF's own formatter produces for 512x768.
|
||||
hf_processor = SimpleNamespace(grid_bos_token="<sop>", grid_eos_token="<eop>", bos_token="<image_start>")
|
||||
tokenizer = SimpleNamespace(encode=lambda text, **kwargs: list(text.encode()))
|
||||
processor.info = SimpleNamespace(
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
|
||||
@@ -341,6 +341,30 @@ class GlmImageDummyInputsBuilder(BaseDummyInputsBuilder[GlmImageProcessingInfo])
|
||||
}
|
||||
|
||||
|
||||
_GLM_IMAGE_GRID_FACTOR = 32
|
||||
|
||||
|
||||
def _build_target_shape_scaffold(processor, *, height: int, width: int, is_text_to_image: bool) -> str:
|
||||
"""Return the grid scaffold HF appends to a GLM-Image generation prompt.
|
||||
|
||||
Mirrors ``GlmImageProcessor._build_prompt_with_target_shape`` (identical in
|
||||
transformers 5.13 and 5.14) using only the processor's public token
|
||||
attributes, so this does not depend on a private HF method that a stub or
|
||||
a future release may not provide.
|
||||
"""
|
||||
factor = _GLM_IMAGE_GRID_FACTOR
|
||||
token_h = (height // factor * factor) // factor
|
||||
token_w = (width // factor * factor) // factor
|
||||
grid_bos, grid_eos, bos = processor.grid_bos_token, processor.grid_eos_token, processor.bos_token
|
||||
scaffold = f"{grid_bos}{token_h} {token_w}{grid_eos}"
|
||||
if is_text_to_image:
|
||||
ratio = token_h / token_w
|
||||
prev_token_h = int(math.sqrt(ratio) * (factor // 2))
|
||||
prev_token_w = int(math.sqrt(1 / ratio) * (factor // 2))
|
||||
scaffold += f"{grid_bos}{prev_token_h} {prev_token_w}{grid_eos}"
|
||||
return scaffold + bos
|
||||
|
||||
|
||||
class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo]):
|
||||
"""
|
||||
Multimodal processor for GLM-Image.
|
||||
@@ -374,12 +398,13 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
# Reuse HF's shape formatter without decoding/re-tokenizing the user's
|
||||
# tokens. Text-to-image needs both target and preview grids; i2i needs
|
||||
# only the target. Grid metadata alone does not supply these AR tokens.
|
||||
# Append HF's target-shape scaffold without decoding/re-tokenizing the
|
||||
# user's tokens. Text-to-image needs both target and preview grids; i2i
|
||||
# needs only the target. Grid metadata alone does not supply these AR
|
||||
# tokens.
|
||||
target_grid = self._build_generation_grids(inputs.hf_processor_mm_kwargs)[0]
|
||||
suffix, *_ = processor._build_prompt_with_target_shape(
|
||||
"",
|
||||
suffix = _build_target_shape_scaffold(
|
||||
processor,
|
||||
height=int(target_grid[1]) * 32,
|
||||
width=int(target_grid[2]) * 32,
|
||||
is_text_to_image=num_images == 0,
|
||||
|
||||
Reference in new issue
Block a user