Merge remote-tracking branch 'origin/main' into dev/vllm-align
Made-with: Cursor # Conflicts: # vllm_omni/entrypoints/openai/serving_chat.py # vllm_omni/worker/gpu_model_runner.py Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
3e539fcb6a
37 files changed
+1072
-169
No files matched your search
@@ -421,6 +421,14 @@ class GPUARModelRunner(OmniGPUModelRunner):
|
||||
if grammar_output is not None:
|
||||
apply_grammar_bitmask(scheduler_output, grammar_output, self.input_batch, logits)
|
||||
|
||||
# Correct padding values of prompt_token_ids to match the logits vocabulary size
|
||||
if logits is not None and not self.input_batch.sampling_metadata.no_penalties:
|
||||
smd = self.input_batch.sampling_metadata
|
||||
if smd.prompt_token_ids is not None:
|
||||
logits_vocab = logits.shape[-1]
|
||||
if self.input_batch.vocab_size > logits_vocab:
|
||||
smd.prompt_token_ids = smd.prompt_token_ids.clamp(max=logits_vocab)
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: sample"):
|
||||
sampler_output = self._sample(logits, spec_decode_metadata)
|
||||
|
||||
@@ -601,7 +609,7 @@ class GPUARModelRunner(OmniGPUModelRunner):
|
||||
if not req_state:
|
||||
return req_id
|
||||
|
||||
add_info = getattr(req_state, "additional_information_cpu", {}) or {}
|
||||
add_info = self.model_intermediate_buffer.get(req_id, {})
|
||||
global_id = add_info.get("global_request_id")
|
||||
if global_id:
|
||||
if isinstance(global_id, list) and global_id:
|
||||
|
||||
Reference in new issue
Block a user