[Model] Optimize CosyVoice3 TensorRT stream handoff (#5673)

Signed-off-by: Allen Wu <allenwu2795@gmail.com>
Co-authored-by: TRAE CLI <noreply@bytedance.com>
Co-authored-by: Sy03 <1370724210@qq.com>
Co-authored-by: TRAE CLI <traecli@bytedance.com>
This commit is contained in:
authored and GitHub committed 2026-08-25 22:51:46 -07:00
1 parent 5de6f7c37a
commit f90e992dc1
4 files changed
+309 -6

No files matched your search

@@ -148,9 +148,9 @@ class ConditionalCFM(BASECFM):
# ``.contiguous().data_ptr()`` could free the temp -> dangling ptr).
io_dtype = getattr(self.estimator, "io_dtype", x.dtype)
[estimator, stream], trt_engine = self.estimator.acquire_estimator()
# NOTE need to synchronize when switching stream
torch.cuda.current_stream().synchronize()
with stream:
caller_stream = torch.cuda.current_stream(x.device)
stream.wait_stream(caller_stream)
with torch.cuda.stream(stream):
x_e = x.to(io_dtype).contiguous()
mask_e = mask.to(io_dtype).contiguous()
mu_e = mu.to(io_dtype).contiguous()
@@ -176,8 +176,13 @@ class ConditionalCFM(BASECFM):
for i, j in enumerate(data_ptrs):
estimator.set_tensor_address(trt_engine.get_tensor_name(i), j)
# run trt engine
assert estimator.execute_async_v3(torch.cuda.current_stream().cuda_stream) is True
torch.cuda.current_stream().synchronize()
assert estimator.execute_async_v3(stream.cuda_stream) is True
for tensor in (x_e, mask_e, mu_e, t_e, spks_e, cond_e, out_e):
if tensor.is_cuda:
tensor.record_stream(stream)
caller_stream.wait_stream(stream)
if out_e.is_cuda:
out_e.record_stream(caller_stream)
self.estimator.release_estimator(estimator, stream)
return out_e.to(x.dtype)
@@ -121,7 +121,7 @@ class TrtContextWrapper:
for _ in range(trt_concurrent):
ctx = engine.create_execution_context()
assert ctx is not None, "failed to create TRT execution context (out of memory?)"
stream = torch.cuda.stream(torch.cuda.Stream(torch.device(device)))
stream = torch.cuda.Stream(torch.device(device))
self._pool.put([ctx, stream])
def acquire_estimator(self):