[BugFix]: fix a lot of bug (#1565)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏 authored and GitHub committed 2026-02-28 16:26:56 +08:00
1 parent cd2234a1bf
commit 3d9fa8d53f
5 files changed
+18 -16

No files matched your search

@@ -128,13 +128,10 @@ class SiglipNaViTWrapper(nn.Module):
else:
self.vision_model = vision_model
# Configure weights for linear equivalent of patch embedding
self.patch_embed_weight = self.vision_model.embeddings.patch_embedding.weight
self.patch_embed_bias = self.vision_model.embeddings.patch_embedding.bias
def forward(self, packed_pixel_values, packed_flattened_position_ids, cu_seqlens, max_seqlen):
w = self.patch_embed_weight.view(self.patch_embed_weight.shape[0], -1)
x = F.linear(packed_pixel_values, w, self.patch_embed_bias)
patch_embed = self.vision_model.embeddings.patch_embedding
w = patch_embed.weight.view(patch_embed.weight.shape[0], -1)
x = F.linear(packed_pixel_values, w, patch_embed.bias)
pos = self.vision_model.embeddings.position_embedding(packed_flattened_position_ids)
x = x + pos
hidden_states = x.unsqueeze(0)
@@ -205,6 +202,9 @@ class BagelPipeline(nn.Module):
)
vit_config_path = os.path.join(model_path, "vit_config.json")
vit_conf = SiglipVisionConfig.from_json_file(vit_config_path)
if vit_conf.num_hidden_layers == 27:
vit_conf.num_hidden_layers = 26
vit_conf.vision_use_head = False
self.vit_model = SiglipVisionModel(vit_conf)
self.image_processor = SiglipImageProcessor.from_pretrained(model_path, local_files_only=True)
@@ -871,8 +871,8 @@ class GlmImagePipeline(nn.Module):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load transformer weights."""
# Filter weights for transformer only
transformer_weights = (
(name.replace("transformer.", ""), weight) for name, weight in weights if name.startswith("transformer.")
(name.replace("transformer.", "", 1), weight) for name, weight in weights if name.startswith("transformer.")
)
return self.transformer.load_weights(transformer_weights)
loaded = self.transformer.load_weights(transformer_weights)
return {f"transformer.{name}" for name in loaded}