[Perf][MOSS-TTS] Avoid synchronization in batched index transfers
Construct MTP row offsets and mixed prefill/decode stop indices on the CPU, then enqueue nonblocking device transfers before batched writes. Add CUDA regressions for synchronization, row placement, stop flags, and snapshot ownership. Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
251ef4f48f
commit
dcd6588a36
3 files changed
+60
-2
No files matched your search
@@ -1971,7 +1971,11 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if start_offsets == list(range(decode_batch_size)):
|
||||
inputs_embeds[:decode_batch_size].copy_(req_embeds[:decode_batch_size])
|
||||
else:
|
||||
offsets = torch.tensor(start_offsets, device=inputs_embeds.device, dtype=torch.long)
|
||||
# A device tensor constructor waits for the preceding MTP graph.
|
||||
# Enqueue the small host index transfer without synchronizing it.
|
||||
offsets = torch.tensor(start_offsets, device="cpu", dtype=torch.long).to(
|
||||
inputs_embeds.device, non_blocking=True
|
||||
)
|
||||
inputs_embeds.index_copy_(0, offsets, req_embeds[:decode_batch_size])
|
||||
if code_predictor_codes is not None:
|
||||
if out_key in getattr(self.model, "gpu_resident_buffer_keys", set()):
|
||||
|
||||
Reference in new issue
Block a user