Benchmark and Bugfix for GLM-Image (#3024)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Jared WenandHongsheng Liu authored and GitHub committed 2026-04-25 20:49:19 +08:00
1 parent 6b52db9e2b
commit 6e35bfbbae
18 files changed
+1836 -381

No files matched your search

@@ -330,6 +330,13 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
- Grid dimension calculation for M-RoPE position encoding
"""
def _cached_apply_hf_processor(self, inputs, timing_ctx):
# i2i: prompt text must be modified based on mm data presence,
# and grid computation requires all images together — bypass cache.
if inputs.mm_data_items.get_all_counts().get("image", 0) > 0:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _call_hf_processor(
self,
prompt: str,
@@ -678,7 +685,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
try:
mrope_grid_thw = self._build_generation_grids(hf_processor_mm_kwargs)
mm_processed_data["mrope_image_grid_thw"] = mrope_grid_thw
logger.info(
logger.debug(
"_apply_hf_processor_main t2i: mrope_image_grid_thw=%s",
mrope_grid_thw.tolist(),
)
@@ -786,7 +793,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
hf_config = self.info.get_hf_config()
image_token_id = getattr(hf_config, "image_token_id", 167855)
image_token_count = prompt_ids.count(image_token_id)
logger.info(
logger.debug(
"_apply_hf_processor_main i2i(HF): num_images=%s, prompt_len=%s, image_token_count=%s, "
"source_grid_shape=%s, mrope_grid_shape=%s",
num_images,
@@ -2327,7 +2334,7 @@ class GlmImageModel(nn.Module):
# Debug: log prior_token_image_ids_info
shapes = [t.shape for t in upsampled_token_ids]
logger.info(
logger.debug(
f"[GlmImageModel.forward] Built prior_token_image_ids_info: "
f"num_images={len(upsampled_token_ids)}, shapes={shapes}, "
f"image_grid_thw={image_grid_thw.tolist()}"
@@ -2537,7 +2544,7 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
# Debug: log prior_token_info
shapes = [t.shape for t in upsampled_token_ids]
logger.info(
logger.debug(
f"[_process_image_input] Built prior_token_info: "
f"num_images={len(upsampled_token_ids)}, shapes={shapes}, "
f"image_grid_thw={image_grid_thw.tolist()}"