[Perf] Optimise tensor concat perf in output processor (#467)

Signed-off-by: wuhang <wuhang6@huawei.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
wuhangandHongsheng Liu authored and GitHub committed 2025-12-25 15:54:41 +00:00
1 parent beb474b65c
commit b3d212e985
1 file changed
+47 -22
+47 -22
View File
@@ -107,22 +107,23 @@ class OmniRequestState(RequestState):
def _to_cpu(x):
if isinstance(x, torch.Tensor):
try:
return x.detach().to("cpu").contiguous()
return x.detach().to("cpu", non_blocking=True).contiguous()
except Exception:
return x
return x
if isinstance(payload, dict):
incoming: Dict[str, Any] = {}
# Optional remap: if producer used "model_outputs" or "hidden", rename to mm_type
# to keep a consistent key namespace per engine_core_output_type.
remapped_tensor = dict(payload)
target_key = self.mm_type or "hidden"
if "model_outputs" in remapped_tensor:
remapped_tensor[target_key] = remapped_tensor.pop("model_outputs")
elif "hidden" in remapped_tensor and target_key != "hidden":
remapped_tensor[target_key] = remapped_tensor.pop("hidden")
for k, v in remapped_tensor.items():
# Iterate directly without unnecessary dict copy
for k, v in payload.items():
# Optional remap: if producer used "model_outputs" or "hidden", rename to mm_type
if k == "model_outputs":
k = target_key
elif k == "hidden" and target_key != "hidden":
k = target_key
if isinstance(v, dict):
incoming[k] = {str(sk): _to_cpu(sv) for sk, sv in v.items()}
else:
@@ -134,29 +135,27 @@ class OmniRequestState(RequestState):
if self.mm_accumulated is None:
self.mm_accumulated = incoming
else:
# Merge keys; concatenate tensors along token dim when possible
# Merge keys; accumulate tensors in lists for deferred concatenation
for k, v in incoming.items():
if k not in self.mm_accumulated:
self.mm_accumulated[k] = v
else:
existing = self.mm_accumulated[k]
if isinstance(v, torch.Tensor) and isinstance(existing, torch.Tensor):
try:
self.mm_accumulated[k] = torch.cat([existing, v], dim=0) # type: ignore[index]
except Exception:
self.mm_accumulated[k] = v
# Use list accumulation to avoid O(n²) repeated concatenation
self.mm_accumulated[k] = [existing, v]
elif isinstance(v, torch.Tensor) and isinstance(existing, list):
# Append to existing list
existing.append(v)
elif isinstance(v, dict) and isinstance(existing, dict):
# Merge nested dicts by concatenating tensors along token dim when possible
# Merge nested dicts with list accumulation for tensors
for sk, sv in v.items():
if sk not in existing:
existing[sk] = sv
continue
ev = existing[sk]
if isinstance(sv, torch.Tensor) and isinstance(ev, torch.Tensor):
try:
existing[sk] = torch.cat([ev, sv], dim=0)
except Exception:
existing[sk] = sv
elif isinstance(sv, torch.Tensor) and isinstance(existing[sk], torch.Tensor):
existing[sk] = [existing[sk], sv]
elif isinstance(sv, torch.Tensor) and isinstance(existing[sk], list):
existing[sk].append(sv)
else:
existing[sk] = sv
else:
@@ -165,6 +164,28 @@ class OmniRequestState(RequestState):
# Log and continue without crashing the output pipeline
logger.exception("Error accumulating multimodal tensor")
def _consolidate_multimodal_tensors(self) -> None:
"""Consolidate accumulated tensor lists into single tensors via concatenation."""
if self.mm_accumulated is None:
return
try:
for k, v in self.mm_accumulated.items():
if isinstance(v, list) and v and isinstance(v[0], torch.Tensor):
try:
self.mm_accumulated[k] = torch.cat(v, dim=0)
except Exception:
# Keep last tensor on failure
self.mm_accumulated[k] = v[-1]
elif isinstance(v, dict):
for sk, sv in v.items():
if isinstance(sv, list) and sv and isinstance(sv[0], torch.Tensor):
try:
v[sk] = torch.cat(sv, dim=0)
except Exception:
v[sk] = sv[-1]
except Exception:
logger.exception("Error consolidating multimodal tensors")
# Override: do not route to pooling-only path; always create completion
# outputs, and attach pooling_result into the CompletionOutput.
def make_request_output(
@@ -417,6 +438,10 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
# Do not consume pooling path now; keep ids and attach mm later
pooling_for_make = None
# Consolidate any accumulated tensor lists before creating output
if isinstance(req_state, OmniRequestState) and finish_reason is not None:
req_state._consolidate_multimodal_tensors()
ro = req_state.make_request_output(
new_token_ids,
pooling_for_make,