[Bug] [Bagel] Fix kv transfer bug (#1437)

Signed-off-by: Ding Zuhao <e1583181@u.nus.edu>
Co-authored-by: Wang Zhipeng: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
Ding ZuhaoandWang Zhipeng: princepride authored and GitHub committed 2026-02-23 15:58:22 +00:00
1 parent 40f72bb870
commit d6a3551a62
1 file changed
+8 -2
@@ -1286,8 +1286,14 @@ class Bagel(nn.Module):
@staticmethod
def _merge_naive_caches(caches: list) -> NaiveCache:
"""Merge multiple NaiveCache objects by concatenating KV tensors per layer."""
merged = NaiveCache(caches[0].num_layers)
for layer_idx in range(merged.num_layers):
if not caches:
# Handle empty list case gracefully if desired,
# though original code also crashed on this.
return NaiveCache(0)
num_layers = len(caches[0].key_cache)
merged = NaiveCache(num_layers)
for layer_idx in range(num_layers):
merged.key_cache[layer_idx] = torch.cat([c.key_cache[layer_idx] for c in caches], dim=0)
merged.value_cache[layer_idx] = torch.cat([c.value_cache[layer_idx] for c in caches], dim=0)
return merged