[Rebase] Rebase to vllm v0.19.0 (#2475)

This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-04-04 15:03:32 +08:00
1 parent 4c031580cf
commit 2804a85e58
52 files changed
+437 -142

No files matched your search

+58 -19
View File
@@ -243,7 +243,7 @@ class OmniGPUModelRunner(GPUModelRunner):
mrope_pos_ptr += completion_part_len
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
def _update_states(self, scheduler_output: "SchedulerOutput"):
"""Update the cached states and the persistent batch with the scheduler
output.
@@ -271,7 +271,7 @@ class OmniGPUModelRunner(GPUModelRunner):
# Zero GPU memory for freshly allocated cache blocks to prevent
# stale NaN/data from corrupting attention or SSM computation.
if hasattr(scheduler_output, "new_block_ids_to_zero") and scheduler_output.new_block_ids_to_zero:
if scheduler_output.new_block_ids_to_zero:
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
# Free the cached encoder outputs.
@@ -300,7 +300,11 @@ class OmniGPUModelRunner(GPUModelRunner):
for req_id in unscheduled_req_ids:
self.input_batch.remove_request(req_id)
if self.use_async_spec_decode:
self.prev_num_draft_tokens.np.fill(0)
reqs_to_add: list[CachedRequestState] = []
deferred_spec_decode_corrections = []
# Add new requests to the cached states.
for new_req_data in scheduler_output.scheduled_new_reqs:
req_id = new_req_data.req_id
@@ -398,10 +402,6 @@ class OmniGPUModelRunner(GPUModelRunner):
req_data = scheduler_output.scheduled_cached_reqs
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
# Wait until valid_sampled_tokens_count is copied to cpu,
# then use it to update actual num_computed_tokens of each request.
valid_sampled_token_count = self._get_valid_sampled_token_count()
for i, req_id in enumerate(req_data.req_ids):
req_state = self.requests[req_id]
num_computed_tokens = req_data.num_computed_tokens[i]
@@ -427,12 +427,18 @@ class OmniGPUModelRunner(GPUModelRunner):
if req_index is None:
req_state.prev_num_draft_len = 0
else:
assert self.input_batch.prev_req_id_to_index is not None
prev_req_index = self.input_batch.prev_req_id_to_index[req_id]
num_accepted = valid_sampled_token_count[prev_req_index] - 1
num_rejected = req_state.prev_num_draft_len - num_accepted
num_computed_tokens -= num_rejected
req_state.output_token_ids.extend([-1] * num_accepted)
optimistic_num_accepted = req_state.prev_num_draft_len
req_state.output_token_ids.extend([-1] * optimistic_num_accepted)
deferred_spec_decode_corrections.append((req_id, optimistic_num_accepted, req_state))
prev_req_index = (
self.input_batch.prev_req_id_to_index.get(req_id)
if self.input_batch.prev_req_id_to_index
else None
)
if prev_req_index is not None:
self.prev_num_draft_tokens.np[prev_req_index] = optimistic_num_accepted
# Update the cached states.
req_state.num_computed_tokens = num_computed_tokens
@@ -449,7 +455,8 @@ class OmniGPUModelRunner(GPUModelRunner):
req_state.output_token_ids.extend(new_token_ids[-num_new_tokens:])
elif num_output_tokens < len(req_state.output_token_ids):
# Some output tokens were discarded due to a sync-KV-load
# failure. Align the cached state.
# failure, or output_token_ids was inflated by the optimistic
# extend above (async spec decode). Align the cached state.
del req_state.output_token_ids[num_output_tokens:]
if req_index is not None:
end_idx = self.input_batch.num_prompt_tokens[req_index] + num_output_tokens
@@ -513,6 +520,35 @@ class OmniGPUModelRunner(GPUModelRunner):
# Refresh batch metadata with any pending updates.
self.input_batch.refresh_metadata()
if deferred_spec_decode_corrections:
def correct_spec_decode_token_counts():
valid_sampled_token_count = self._get_valid_sampled_token_count()
if not valid_sampled_token_count:
return
prev_req_id_to_index = self.input_batch.prev_req_id_to_index
if not prev_req_id_to_index:
return
for (
req_id,
optimistic_num_accepted,
req_state,
) in deferred_spec_decode_corrections:
prev_req_index = prev_req_id_to_index.get(req_id)
if prev_req_index is None:
continue
num_accepted = valid_sampled_token_count[prev_req_index] - 1
correction = optimistic_num_accepted - num_accepted
req_state.num_computed_tokens -= correction
cur_req_index = self.input_batch.req_id_to_index.get(req_id)
if cur_req_index is None:
continue
self.input_batch.num_computed_tokens_cpu[cur_req_index] -= correction
return correct_spec_decode_token_counts
else:
return None
@torch.inference_mode()
def extract_multimodal_outputs(self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput) -> dict:
if (
@@ -704,11 +740,14 @@ class OmniGPUModelRunner(GPUModelRunner):
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] # type: ignore[assignment]
else:
seq_lens = max_query_len # type: ignore[assignment]
self.seq_lens.np[:num_reqs] = seq_lens
self.seq_lens.np[num_reqs:] = 0
self.seq_lens.copy_to_gpu()
self.seq_lens[:num_reqs] = (
seq_lens
if isinstance(seq_lens, int)
else torch.tensor(seq_lens, dtype=torch.int32, device=self.device)
)
self.seq_lens[num_reqs:] = 0
cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens)
cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self._arange_scratch)
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
self.query_start_loc.copy_to_gpu()
@@ -759,7 +798,7 @@ class OmniGPUModelRunner(GPUModelRunner):
elif self.uses_xdrope_dim > 0:
positions = self.xdrope_positions.gpu[:, :num_tokens_padded]
else:
positions = self.positions.gpu[:num_tokens_padded]
positions = self.positions[:num_tokens_padded]
if get_pp_group().is_first_rank:
intermediate_tensors = None
@@ -1164,7 +1203,7 @@ class OmniGPUModelRunner(GPUModelRunner):
elif self.uses_xdrope_dim > 0:
positions = self.xdrope_positions.gpu[:, :num_input_tokens]
else:
positions = self.positions.gpu[:num_input_tokens]
positions = self.positions[:num_input_tokens]
if is_first_rank:
intermediate_tensors = None