Fix NPU code predictor device mismatch in concurrent mode (#3453)
Co-authored-by: houzechen <h00875519@china.huawei.com>
This commit is contained in:
1 parent
f6e3dece09
commit
d460673647
1 file changed
+1
-2
@@ -460,7 +460,6 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
**talker_kwargs,
|
||||
)
|
||||
# update the inputs_embeds and code_predictor_codes
|
||||
code_predictor_codes_cpu = code_predictor_codes.detach().to("cpu").contiguous()
|
||||
out_key = getattr(self.model, "talker_mtp_output_key", ("codes", "audio"))
|
||||
if not isinstance(out_key, tuple) or len(out_key) != 2:
|
||||
raise TypeError(f"talker_mtp_output_key must be a 2-tuple, got {type(out_key).__name__}: {out_key!r}")
|
||||
@@ -468,5 +467,5 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
req_index = self.input_batch.req_ids.index(req_id)
|
||||
start_offset = int(self.query_start_loc.cpu[req_index])
|
||||
inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1]
|
||||
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes_cpu[idx : idx + 1]}}
|
||||
update_dict = {out_key[0]: {out_key[1]: code_predictor_codes[idx : idx + 1]}}
|
||||
self._merge_additional_information_update(req_id, update_dict)
|
||||
Reference in new issue
Block a user