rebase: adapt to vLLM v0.29 APIs
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
6a63f5cb99
commit
87a1fd8da2
60 files changed
+2038
-514
No files matched your search
@@ -48,7 +48,7 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in
|
||||
if not min_toks:
|
||||
continue
|
||||
needs_rebuild = False
|
||||
for _, _, stop_tok_ids in min_toks.values():
|
||||
for _, _, stop_tok_ids, _ in min_toks.values():
|
||||
oob = [tok for tok in stop_tok_ids if tok >= logits_vocab]
|
||||
if not oob:
|
||||
continue
|
||||
@@ -63,10 +63,19 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in
|
||||
if needs_rebuild:
|
||||
reqs: list[int] = []
|
||||
tok_ids: list[int] = []
|
||||
for index, (_, _, stop_tok_ids) in min_toks.items():
|
||||
restore_reqs: list[int] = []
|
||||
restore_tok_ids: list[int] = []
|
||||
for index, (_, _, stop_tok_ids, uses_structured_output) in min_toks.items():
|
||||
reqs.extend([index] * len(stop_tok_ids))
|
||||
tok_ids.extend(stop_tok_ids)
|
||||
if uses_structured_output:
|
||||
restore_reqs.extend([index] * len(stop_tok_ids))
|
||||
restore_tok_ids.extend(stop_tok_ids)
|
||||
proc.logits_slice = (
|
||||
proc._device_tensor(reqs, torch.int32),
|
||||
proc._device_tensor(tok_ids, torch.int32),
|
||||
)
|
||||
proc.restore_logits_slice = (
|
||||
proc._device_tensor(restore_reqs, torch.int32),
|
||||
proc._device_tensor(restore_tok_ids, torch.int32),
|
||||
)
|
||||
Reference in new issue
Block a user