[Bugfix] MiniCPMo-4.5 bugfix (#5233)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: Guanzhe (Leo) Huang <116847811+leohuang257@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
2 files changed
+186
-2
No files matched your search
@@ -3175,9 +3175,33 @@ class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
|
||||
|
||||
|
||||
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
audio_features = hf_inputs.get("audio_features")
|
||||
audio_feature_lens = hf_inputs.get("audio_feature_lens")
|
||||
audio_features_cfg = MultiModalFieldConfig.batched("audio")
|
||||
|
||||
if audio_features is not None and audio_feature_lens is not None:
|
||||
num_features = len(audio_features)
|
||||
num_audios = len(audio_feature_lens)
|
||||
|
||||
if num_features > num_audios:
|
||||
chunks_per_audio = [lens.numel() if isinstance(lens, torch.Tensor) else 1 for lens in audio_feature_lens]
|
||||
if sum(chunks_per_audio) != num_features:
|
||||
chunks_per_audio = [
|
||||
max(int((lens != 0).sum()), 1) if isinstance(lens, torch.Tensor) else 1
|
||||
for lens in audio_feature_lens
|
||||
]
|
||||
|
||||
slice_idxs = [0]
|
||||
for num_chunks in chunks_per_audio:
|
||||
slice_idxs.append(slice_idxs[-1] + num_chunks)
|
||||
audio_features_cfg = MultiModalFieldConfig.flat(
|
||||
"audio",
|
||||
[slice(slice_idxs[i], slice_idxs[i + 1]) for i in range(len(chunks_per_audio))],
|
||||
)
|
||||
|
||||
return dict(
|
||||
**_minicpmv_field_config(hf_inputs),
|
||||
audio_features=MultiModalFieldConfig.batched("audio"),
|
||||
audio_features=audio_features_cfg,
|
||||
audio_feature_lens=MultiModalFieldConfig.batched("audio"),
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
)
|
||||
@@ -3369,11 +3393,17 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
|
||||
out_keys={"audio_features", "audio_feature_lens"},
|
||||
)
|
||||
|
||||
flat_feature_lens: list[int] = []
|
||||
for lens in audio_inputs["audio_feature_lens"]:
|
||||
if isinstance(lens, torch.Tensor):
|
||||
flat_feature_lens.extend(lens.flatten().tolist())
|
||||
else:
|
||||
flat_feature_lens.append(int(lens))
|
||||
unpadded_audio_features = [
|
||||
feat[:, :feature_len]
|
||||
for feat, feature_len in zip(
|
||||
audio_inputs["audio_features"],
|
||||
audio_inputs["audio_feature_lens"],
|
||||
flat_feature_lens,
|
||||
)
|
||||
]
|
||||
audio_inputs["audio_features"] = unpadded_audio_features
|
||||
|
||||
Reference in new issue
Block a user