Fix NPU code predictor device mismatch in concurrent mode (#3453)

Co-authored-by: houzechen <h00875519@china.huawei.com>
This commit is contained in:
Wallbreazzzandhouzechen authored and GitHub committed 2026-05-09 14:58:56 +08:00
1 parent f6e3dece09
commit d460673647
1 file changed
+1 -2
@@ -460,7 +460,6 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
**talker_kwargs,
)
# update the inputs_embeds and code_predictor_codes
code_predictor_codes_cpu = code_predictor_codes.detach().to("cpu").contiguous()
out_key = getattr(self.model, "talker_mtp_output_key", ("codes", "audio"))
if not isinstance(out_key, tuple) or len(out_key) != 2:
raise TypeError(f"talker_mtp_output_key must be a 2-tuple, got {type(out_key).__name__}: {out_key!r}")
@@ -468,5 +467,5 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
req_index = self.input_batch.req_ids.index(req_id)
start_offset = int(self.query_start_loc.cpu[req_index])
inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1]
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes_cpu[idx : idx + 1]}}
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes[idx : idx + 1]}}
self._merge_additional_information_update(req_id, update_dict)