Benchmark and Bugfix for GLM-Image (#3024)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
6b52db9e2b
commit
6e35bfbbae
18 files changed
+1836
-381
No files matched your search
@@ -330,6 +330,13 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
- Grid dimension calculation for M-RoPE position encoding
|
||||
"""
|
||||
|
||||
def _cached_apply_hf_processor(self, inputs, timing_ctx):
|
||||
# i2i: prompt text must be modified based on mm data presence,
|
||||
# and grid computation requires all images together — bypass cache.
|
||||
if inputs.mm_data_items.get_all_counts().get("image", 0) > 0:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -678,7 +685,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
try:
|
||||
mrope_grid_thw = self._build_generation_grids(hf_processor_mm_kwargs)
|
||||
mm_processed_data["mrope_image_grid_thw"] = mrope_grid_thw
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"_apply_hf_processor_main t2i: mrope_image_grid_thw=%s",
|
||||
mrope_grid_thw.tolist(),
|
||||
)
|
||||
@@ -786,7 +793,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
hf_config = self.info.get_hf_config()
|
||||
image_token_id = getattr(hf_config, "image_token_id", 167855)
|
||||
image_token_count = prompt_ids.count(image_token_id)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"_apply_hf_processor_main i2i(HF): num_images=%s, prompt_len=%s, image_token_count=%s, "
|
||||
"source_grid_shape=%s, mrope_grid_shape=%s",
|
||||
num_images,
|
||||
@@ -2327,7 +2334,7 @@ class GlmImageModel(nn.Module):
|
||||
|
||||
# Debug: log prior_token_image_ids_info
|
||||
shapes = [t.shape for t in upsampled_token_ids]
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"[GlmImageModel.forward] Built prior_token_image_ids_info: "
|
||||
f"num_images={len(upsampled_token_ids)}, shapes={shapes}, "
|
||||
f"image_grid_thw={image_grid_thw.tolist()}"
|
||||
@@ -2537,7 +2544,7 @@ class GlmImageForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP
|
||||
|
||||
# Debug: log prior_token_info
|
||||
shapes = [t.shape for t in upsampled_token_ids]
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"[_process_image_input] Built prior_token_info: "
|
||||
f"num_images={len(upsampled_token_ids)}, shapes={shapes}, "
|
||||
f"image_grid_thw={image_grid_thw.tolist()}"
|
||||
|
||||
Reference in new issue
Block a user