[Bugfix] Add missing .gpu accessor for inputs_embeds CpuGpuBuffer in prefill overlay (#2068)
Signed-off-by: dubin555 <dubin555@gmail.com>
This commit is contained in:
1 parent
08cf5a6083
commit
dd0fa02547
1 file changed
+1
-1
@@ -1120,7 +1120,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
dtype=self.dtype, device=self.device, non_blocking=True
|
||||
)
|
||||
start_offset = int(self.query_start_loc.cpu[req_index])
|
||||
self.inputs_embeds[start_offset : start_offset + overlay_len].copy_(src)
|
||||
self.inputs_embeds.gpu[start_offset : start_offset + overlay_len].copy_(src)
|
||||
|
||||
def _update_additional_information(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
|
||||
Reference in new issue
Block a user