[Feat]audio streaming input for async chunk (#3614)

Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
ChenWenjing authored and GitHub committed 2026-05-31 17:39:25 +08:00
1 parent 6f4bd3e45c
commit 5dfdf584dd
20 files changed
+536 -184

No files matched your search

+8 -30
View File
@@ -437,7 +437,7 @@ class OmniGPUModelRunner(GPUModelRunner):
for new_req_data in scheduler_output.scheduled_new_reqs:
req_id = new_req_data.req_id
if req_id in self.requests:
self._update_streaming_input_additional_info(new_req_data, req_id)
self._update_streaming_input_additional_info(req_id)
req_state = self._update_streaming_request(req_id, new_req_data)
reqs_to_add.append(req_state)
continue
@@ -1806,34 +1806,12 @@ class OmniGPUModelRunner(GPUModelRunner):
logger.warning_once("_merge_additional_information_update is deprecated, use _update_intermediate_buffer")
return self._update_intermediate_buffer(req_id, upd)
def _update_streaming_input_additional_info(self, new_req_data, req_id):
# For streaming input prefill case only. Update buffer from last segment input
def _update_streaming_input_additional_info(self, req_id):
# For streaming input prefill case only. Set num processed tokens = 0 for new segment input
cached_additional_info = self.model_intermediate_buffer.get(req_id, {})
if cached_additional_info:
payload_info = getattr(new_req_data, "additional_information", None)
inc_info = deserialize_additional_information(payload_info)
if isinstance(inc_info, dict) and inc_info:
accumulated_keys: set[tuple[str, str]] = set()
if hasattr(self, "model") and hasattr(self.model, "streaming_accumulated_keys"):
accumulated_keys = self.model.streaming_accumulated_keys
merged_info = dict(cached_additional_info)
for key, value in inc_info.items():
if isinstance(value, dict):
existing_sub = merged_info.get(key)
merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {}
for sk, sv in value.items():
if (key, sk) in accumulated_keys and isinstance(sv, torch.Tensor):
inc_tensor = sv.detach().to("cpu").contiguous()
old_tensor = merged_sub.get(sk)
if old_tensor is None:
merged_sub[sk] = inc_tensor
else:
merged_sub[sk] = torch.cat((old_tensor, inc_tensor), dim=0)
else:
merged_sub[sk] = sv
merged_info[key] = merged_sub
else:
merged_info[key] = value
merged_info.setdefault("meta", {})["num_processed_tokens"] = 0
self.model_intermediate_buffer[req_id] = merged_info
setattr(self.requests[req_id], "additional_information_cpu", merged_info)
merged_info = dict(cached_additional_info)
merged_info.setdefault("meta", {})["num_processed_tokens"] = 0
merged_info.setdefault("meta", {})["resumable"] = True
self.model_intermediate_buffer[req_id] = merged_info
setattr(self.requests[req_id], "additional_information_cpu", merged_info)