[Minor][Refactor] Pass seq_token_counts explicitly (#1425)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
1589931fa6
commit
af11b02081
3 files changed
+29
-24
No files matched your search
@@ -250,6 +250,9 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
intermediate_tensors,
|
||||
)
|
||||
|
||||
# [Omni] Pass token counts per request for code2wav output slicing
|
||||
model_kwargs["seq_token_counts"] = tokens
|
||||
|
||||
# Set cudagraph mode to none if calc_kv_scales is true.
|
||||
# KV scales calculation involves dynamic operations that are incompatible
|
||||
# with CUDA graph capture.
|
||||
@@ -258,10 +261,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
# Mark KV scales as calculated after the first forward pass
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
if ubatch_slices_padded is None:
|
||||
# reuse ubatch_slices_padded for code2wav batching
|
||||
ubatch_slices_padded = tokens
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
with (
|
||||
|
||||
Reference in new issue
Block a user