Add max_query_len parameter to _LocalPredictorKVCache for improved query handling
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
2da9d28f1b
commit
16824aa295
1 file changed
+1
@@ -176,6 +176,7 @@ class _LocalPredictorKVCache:
|
||||
num_tokens=num_tokens,
|
||||
query_start_loc_gpu=query_start_loc_gpu,
|
||||
query_start_loc_cpu=qsl,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=seq_lens_gpu,
|
||||
max_seq_len=max_seq_len,
|
||||
block_tables=[block_table],
|
||||
|
||||
Reference in new issue
Block a user