[Minor][Refactor] Pass seq_token_counts explicitly (#1425)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Canlin GuoandHongsheng Liu authored and GitHub committed 2026-02-24 14:10:00 +08:00
1 parent 1589931fa6
commit af11b02081
3 files changed
+29 -24

No files matched your search

@@ -250,6 +250,9 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
intermediate_tensors,
)
# [Omni] Pass token counts per request for code2wav output slicing
model_kwargs["seq_token_counts"] = tokens
# Set cudagraph mode to none if calc_kv_scales is true.
# KV scales calculation involves dynamic operations that are incompatible
# with CUDA graph capture.
@@ -258,10 +261,6 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
# Mark KV scales as calculated after the first forward pass
self.calculate_kv_scales = False
if ubatch_slices_padded is None:
# reuse ubatch_slices_padded for code2wav batching
ubatch_slices_padded = tokens
# Run the model.
# Use persistent buffers for CUDA graphs.
with (