[Rebase] Rebase to vllm v0.19.0 (#2475)
This commit is contained in:
1 parent
4c031580cf
commit
2804a85e58
52 files changed
+437
-142
No files matched your search
@@ -243,7 +243,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
mrope_pos_ptr += completion_part_len
|
||||
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput"):
|
||||
"""Update the cached states and the persistent batch with the scheduler
|
||||
output.
|
||||
|
||||
@@ -271,7 +271,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
# Zero GPU memory for freshly allocated cache blocks to prevent
|
||||
# stale NaN/data from corrupting attention or SSM computation.
|
||||
if hasattr(scheduler_output, "new_block_ids_to_zero") and scheduler_output.new_block_ids_to_zero:
|
||||
if scheduler_output.new_block_ids_to_zero:
|
||||
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||
|
||||
# Free the cached encoder outputs.
|
||||
@@ -300,7 +300,11 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
for req_id in unscheduled_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
if self.use_async_spec_decode:
|
||||
self.prev_num_draft_tokens.np.fill(0)
|
||||
|
||||
reqs_to_add: list[CachedRequestState] = []
|
||||
deferred_spec_decode_corrections = []
|
||||
# Add new requests to the cached states.
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
req_id = new_req_data.req_id
|
||||
@@ -398,10 +402,6 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
req_data = scheduler_output.scheduled_cached_reqs
|
||||
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
|
||||
# Wait until valid_sampled_tokens_count is copied to cpu,
|
||||
# then use it to update actual num_computed_tokens of each request.
|
||||
valid_sampled_token_count = self._get_valid_sampled_token_count()
|
||||
|
||||
for i, req_id in enumerate(req_data.req_ids):
|
||||
req_state = self.requests[req_id]
|
||||
num_computed_tokens = req_data.num_computed_tokens[i]
|
||||
@@ -427,12 +427,18 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if req_index is None:
|
||||
req_state.prev_num_draft_len = 0
|
||||
else:
|
||||
assert self.input_batch.prev_req_id_to_index is not None
|
||||
prev_req_index = self.input_batch.prev_req_id_to_index[req_id]
|
||||
num_accepted = valid_sampled_token_count[prev_req_index] - 1
|
||||
num_rejected = req_state.prev_num_draft_len - num_accepted
|
||||
num_computed_tokens -= num_rejected
|
||||
req_state.output_token_ids.extend([-1] * num_accepted)
|
||||
optimistic_num_accepted = req_state.prev_num_draft_len
|
||||
req_state.output_token_ids.extend([-1] * optimistic_num_accepted)
|
||||
|
||||
deferred_spec_decode_corrections.append((req_id, optimistic_num_accepted, req_state))
|
||||
|
||||
prev_req_index = (
|
||||
self.input_batch.prev_req_id_to_index.get(req_id)
|
||||
if self.input_batch.prev_req_id_to_index
|
||||
else None
|
||||
)
|
||||
if prev_req_index is not None:
|
||||
self.prev_num_draft_tokens.np[prev_req_index] = optimistic_num_accepted
|
||||
|
||||
# Update the cached states.
|
||||
req_state.num_computed_tokens = num_computed_tokens
|
||||
@@ -449,7 +455,8 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
req_state.output_token_ids.extend(new_token_ids[-num_new_tokens:])
|
||||
elif num_output_tokens < len(req_state.output_token_ids):
|
||||
# Some output tokens were discarded due to a sync-KV-load
|
||||
# failure. Align the cached state.
|
||||
# failure, or output_token_ids was inflated by the optimistic
|
||||
# extend above (async spec decode). Align the cached state.
|
||||
del req_state.output_token_ids[num_output_tokens:]
|
||||
if req_index is not None:
|
||||
end_idx = self.input_batch.num_prompt_tokens[req_index] + num_output_tokens
|
||||
@@ -513,6 +520,35 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
# Refresh batch metadata with any pending updates.
|
||||
self.input_batch.refresh_metadata()
|
||||
|
||||
if deferred_spec_decode_corrections:
|
||||
|
||||
def correct_spec_decode_token_counts():
|
||||
valid_sampled_token_count = self._get_valid_sampled_token_count()
|
||||
if not valid_sampled_token_count:
|
||||
return
|
||||
prev_req_id_to_index = self.input_batch.prev_req_id_to_index
|
||||
if not prev_req_id_to_index:
|
||||
return
|
||||
for (
|
||||
req_id,
|
||||
optimistic_num_accepted,
|
||||
req_state,
|
||||
) in deferred_spec_decode_corrections:
|
||||
prev_req_index = prev_req_id_to_index.get(req_id)
|
||||
if prev_req_index is None:
|
||||
continue
|
||||
num_accepted = valid_sampled_token_count[prev_req_index] - 1
|
||||
correction = optimistic_num_accepted - num_accepted
|
||||
req_state.num_computed_tokens -= correction
|
||||
cur_req_index = self.input_batch.req_id_to_index.get(req_id)
|
||||
if cur_req_index is None:
|
||||
continue
|
||||
self.input_batch.num_computed_tokens_cpu[cur_req_index] -= correction
|
||||
|
||||
return correct_spec_decode_token_counts
|
||||
else:
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
def extract_multimodal_outputs(self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput) -> dict:
|
||||
if (
|
||||
@@ -704,11 +740,14 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] # type: ignore[assignment]
|
||||
else:
|
||||
seq_lens = max_query_len # type: ignore[assignment]
|
||||
self.seq_lens.np[:num_reqs] = seq_lens
|
||||
self.seq_lens.np[num_reqs:] = 0
|
||||
self.seq_lens.copy_to_gpu()
|
||||
self.seq_lens[:num_reqs] = (
|
||||
seq_lens
|
||||
if isinstance(seq_lens, int)
|
||||
else torch.tensor(seq_lens, dtype=torch.int32, device=self.device)
|
||||
)
|
||||
self.seq_lens[num_reqs:] = 0
|
||||
|
||||
cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens)
|
||||
cum_num_tokens = self._get_cumsum_and_arange(num_scheduled_tokens, self._arange_scratch)
|
||||
self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens
|
||||
self.query_start_loc.copy_to_gpu()
|
||||
|
||||
@@ -759,7 +798,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_tokens_padded]
|
||||
else:
|
||||
positions = self.positions.gpu[:num_tokens_padded]
|
||||
positions = self.positions[:num_tokens_padded]
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
intermediate_tensors = None
|
||||
@@ -1164,7 +1203,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_input_tokens]
|
||||
else:
|
||||
positions = self.positions.gpu[:num_input_tokens]
|
||||
positions = self.positions[:num_input_tokens]
|
||||
|
||||
if is_first_rank:
|
||||
intermediate_tensors = None
|
||||
|
||||
Reference in new issue
Block a user