[Feat]audio streaming input for async chunk (#3614)
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
1 parent
6f4bd3e45c
commit
5dfdf584dd
20 files changed
+536
-184
No files matched your search
@@ -437,7 +437,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
req_id = new_req_data.req_id
|
||||
if req_id in self.requests:
|
||||
self._update_streaming_input_additional_info(new_req_data, req_id)
|
||||
self._update_streaming_input_additional_info(req_id)
|
||||
req_state = self._update_streaming_request(req_id, new_req_data)
|
||||
reqs_to_add.append(req_state)
|
||||
continue
|
||||
@@ -1806,34 +1806,12 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
logger.warning_once("_merge_additional_information_update is deprecated, use _update_intermediate_buffer")
|
||||
return self._update_intermediate_buffer(req_id, upd)
|
||||
|
||||
def _update_streaming_input_additional_info(self, new_req_data, req_id):
|
||||
# For streaming input prefill case only. Update buffer from last segment input
|
||||
def _update_streaming_input_additional_info(self, req_id):
|
||||
# For streaming input prefill case only. Set num processed tokens = 0 for new segment input
|
||||
cached_additional_info = self.model_intermediate_buffer.get(req_id, {})
|
||||
if cached_additional_info:
|
||||
payload_info = getattr(new_req_data, "additional_information", None)
|
||||
inc_info = deserialize_additional_information(payload_info)
|
||||
if isinstance(inc_info, dict) and inc_info:
|
||||
accumulated_keys: set[tuple[str, str]] = set()
|
||||
if hasattr(self, "model") and hasattr(self.model, "streaming_accumulated_keys"):
|
||||
accumulated_keys = self.model.streaming_accumulated_keys
|
||||
merged_info = dict(cached_additional_info)
|
||||
for key, value in inc_info.items():
|
||||
if isinstance(value, dict):
|
||||
existing_sub = merged_info.get(key)
|
||||
merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {}
|
||||
for sk, sv in value.items():
|
||||
if (key, sk) in accumulated_keys and isinstance(sv, torch.Tensor):
|
||||
inc_tensor = sv.detach().to("cpu").contiguous()
|
||||
old_tensor = merged_sub.get(sk)
|
||||
if old_tensor is None:
|
||||
merged_sub[sk] = inc_tensor
|
||||
else:
|
||||
merged_sub[sk] = torch.cat((old_tensor, inc_tensor), dim=0)
|
||||
else:
|
||||
merged_sub[sk] = sv
|
||||
merged_info[key] = merged_sub
|
||||
else:
|
||||
merged_info[key] = value
|
||||
merged_info.setdefault("meta", {})["num_processed_tokens"] = 0
|
||||
self.model_intermediate_buffer[req_id] = merged_info
|
||||
setattr(self.requests[req_id], "additional_information_cpu", merged_info)
|
||||
merged_info = dict(cached_additional_info)
|
||||
merged_info.setdefault("meta", {})["num_processed_tokens"] = 0
|
||||
merged_info.setdefault("meta", {})["resumable"] = True
|
||||
self.model_intermediate_buffer[req_id] = merged_info
|
||||
setattr(self.requests[req_id], "additional_information_cpu", merged_info)
|
||||
Reference in new issue
Block a user