[Bugfix] Fix full-payload mm splitting for dual hidden/scheduled batch axes (#4870 follow-up) (#4910)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
ba7e9e6496
commit
9ceca4ec85
4 files changed
+96
-18
No files matched your search
@@ -839,7 +839,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
end: int,
|
||||
audio_sparse_output: bool,
|
||||
sparse_mm_index: dict[str, int],
|
||||
seq_len: int,
|
||||
hidden_seq_len: int,
|
||||
scheduled_seq_len: int,
|
||||
) -> dict[str, object]:
|
||||
if combined_multimodal_outputs:
|
||||
return self._build_combined_prefix_cache_mm_payload(
|
||||
@@ -876,7 +877,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
start=start,
|
||||
end=end,
|
||||
pass_lists_through=False,
|
||||
seq_len=seq_len,
|
||||
seq_len=hidden_seq_len,
|
||||
scheduled_seq_len=scheduled_seq_len,
|
||||
)
|
||||
return mm_payload
|
||||
|
||||
@@ -894,7 +896,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
mm_cpu: dict[str, object] | None,
|
||||
audio_sparse_output: bool,
|
||||
sparse_mm_index: dict[str, int],
|
||||
seq_len: int,
|
||||
hidden_seq_len: int,
|
||||
scheduled_seq_len: int,
|
||||
) -> dict[str, object]:
|
||||
payload: dict[str, object] = {}
|
||||
if not audio_sparse_output:
|
||||
@@ -920,7 +923,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
end=end,
|
||||
audio_sparse_output=audio_sparse_output,
|
||||
sparse_mm_index=sparse_mm_index,
|
||||
seq_len=seq_len,
|
||||
hidden_seq_len=hidden_seq_len,
|
||||
scheduled_seq_len=scheduled_seq_len,
|
||||
)
|
||||
payload.update(mm_payload)
|
||||
return payload
|
||||
@@ -1731,7 +1735,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
# The actual multimodal wire transport uses multimodal_outputs instead.
|
||||
pooler_output: list[dict[str, object]] | None = None
|
||||
if needs_pooler_payload:
|
||||
mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)
|
||||
hidden_seq_len = int(hidden_states.shape[0])
|
||||
scheduled_seq_len = int(scheduler_output.total_num_scheduled_tokens)
|
||||
mm_cpu = None
|
||||
if self.omni_prefix_cache is not None:
|
||||
(
|
||||
@@ -1787,7 +1792,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
mm_cpu=mm_cpu,
|
||||
audio_sparse_output=audio_sparse_output,
|
||||
sparse_mm_index=sparse_mm_index,
|
||||
seq_len=mm_seq_len,
|
||||
hidden_seq_len=hidden_seq_len,
|
||||
scheduled_seq_len=scheduled_seq_len,
|
||||
)
|
||||
pooler_output.append(flatten_payload(payload))
|
||||
|
||||
|
||||
Reference in new issue
Block a user