[Rebase] Rebase to vllm v0.21.0 (#3530)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk>
Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com>
Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com>
Signed-off-by: Dnoob <dxpouo@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Signed-off-by: rein yang <ruiruyang2@gmail.com>
Signed-off-by: Nick Cao <ncao@redhat.com>
Signed-off-by: zhumingjue <zhumingjue@huawei.com>
Signed-off-by: Ricardo Noriega De Soto <rnoriega@redhat.com>
Signed-off-by: lyj-jjj <liuyingjun5@huawei.com>
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: weizhoublue <weizhoublue@github.com>
Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io>
Signed-off-by: dengyunyang <584797741@qq.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Signed-off-by: David Chen <530634352@qq.com>
Signed-off-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
Signed-off-by: bjf-frz <frz123db@gmail.com>
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Signed-off-by: KexiongYu <yukexiong1@huawei.com>
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: NumberWan <wantszkin2003@gmail.com>
Co-authored-by: dsinghvi <divyanshsinghvi@gmail.com>
Co-authored-by: Dnoob <dxpouo@gmail.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Co-authored-by: Samit <285365963@qq.com>
Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com>
Co-authored-by: Nick Cao <ncao@redhat.com>
Co-authored-by: zhumingjue138 <zhumingjue@huawei.com>
Co-authored-by: Ricardo Noriega <rnoriega@redhat.com>
Co-authored-by: lyj-jjj <liuyingjun5@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Co-authored-by: weizhoublue <45163302+weizhoublue@users.noreply.github.com>
Co-authored-by: weizhoublue <weizhoublue@github.com>
Co-authored-by: dengyunyang <584797741@qq.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
Co-authored-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: NATURE <wzliu@connect.hku.hk>
Co-authored-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Y. Fisher <yukexiong1@huawei.com>
Co-authored-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
46 files changed
+589 -258

No files matched your search

+37 -12
View File
@@ -20,7 +20,9 @@ from vllm.distributed.parallel_state import get_pp_group, get_tp_group
from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
extract_routed_experts_for_current_batch,
get_global_experts_capturer,
issue_routing_d2h_copy,
)
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
@@ -28,6 +30,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
from vllm.v1.structured_output.utils import apply_grammar_bitmask
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker.gpu_model_runner import (
@@ -358,11 +361,21 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
self.flush_full_payload_outputs(flush_ids)
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.clear_buffer() # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
capturer = get_global_experts_capturer()
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
capturer.finalize_pending_copy()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
# The replace is much faster than deepcopy.
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
spec_decode_tokens_copy = scheduler_output.scheduled_spec_decode_tokens.copy()
scheduler_output = replace(
scheduler_output,
num_scheduled_tokens=num_scheduled_tokens_copy,
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
)
if has_kv_transfer_group():
kv_connector_metadata = scheduler_output.kv_connector_metadata
@@ -678,6 +691,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
if self.routed_experts_initialized and hasattr(self, "_positions_cpu"):
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
return None
def _sample(
@@ -829,7 +850,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
if use_gpu_toks:
assert isinstance(
self.drafter,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
)
sampled_token_ids = sampler_output.sampled_token_ids
if input_fits_in_drafter:
@@ -1013,12 +1034,15 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
self.accumulate_full_payload_output(rid, pooler_output[i], req_state)
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
routed_experts_dict = None
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
output = OmniModelRunnerOutput(
req_ids=req_ids_output_copy,
req_id_to_index=req_id_to_index_output_copy,
@@ -1030,6 +1054,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
num_nans_in_logits=num_nans_in_logits,
cudagraph_stats=cudagraph_stats,
routed_experts_dict=routed_experts_dict,
)
output.kv_extracted_req_ids = kv_extracted_req_ids
output.omni_connector_output = self.get_omni_connector_output()
@@ -9,6 +9,7 @@ from __future__ import annotations
import gc
import logging
from copy import copy
from dataclasses import replace
import numpy as np
import torch
@@ -18,7 +19,9 @@ from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_
from vllm.distributed.parallel_state import get_pp_group
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
extract_routed_experts_for_current_batch,
get_global_experts_capturer,
issue_routing_d2h_copy,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
@@ -27,6 +30,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker.gpu_model_runner import (
EMPTY_MODEL_RUNNER_OUTPUT,
@@ -111,11 +115,21 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
self.flush_full_payload_outputs(flush_ids)
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.clear_buffer() # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
capturer = get_global_experts_capturer()
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
capturer.finalize_pending_copy()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
# The replace is much faster than deepcopy.
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
spec_decode_tokens_copy = scheduler_output.scheduled_spec_decode_tokens.copy()
scheduler_output = replace(
scheduler_output,
num_scheduled_tokens=num_scheduled_tokens_copy,
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
)
if has_kv_transfer_group():
kv_connector_metadata = scheduler_output.kv_connector_metadata
@@ -339,6 +353,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
if self.routed_experts_initialized and hasattr(self, "_positions_cpu"):
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
return None
@torch.inference_mode()
@@ -425,6 +447,15 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
# [Omni] Copy req_id mappings to avoid async scheduling mutation.
req_ids_output_copy = self.input_batch.req_ids.copy()
req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy()
routed_experts_dict = None
if self.routed_experts_initialized:
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
if self._should_accumulate_full_payload_output():
for i, rid in enumerate(req_ids_output_copy):
req_state = self.requests.get(rid)
@@ -442,6 +473,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
num_nans_in_logits={},
cudagraph_stats=cudagraph_stats,
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
routed_experts_dict=routed_experts_dict,
)
output.omni_connector_output = self.get_omni_connector_output()
@@ -751,7 +783,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
device=self.device,
)
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens_padded, None, False)
intermediate_tensors = self.sync_and_gather_intermediate_tensors(num_tokens_padded, None, False)
if ubatch_slices_padded is not None:
# Adjust values to reflect a single ubatch.
@@ -794,7 +826,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
):
assert isinstance(
self.drafter,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
)
assert self.speculative_config is not None
# Eagle currently only supports PIECEWISE cudagraphs.
+59 -10
View File
@@ -1,4 +1,5 @@
import contextlib
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast
import numpy as np
@@ -18,6 +19,11 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
from vllm.v1.spec_decode.ngram_proposer_gpu import (
update_ngram_gpu_tensors_incremental,
update_scheduler_for_invalid_drafts,
)
from vllm.v1.worker.gpu_input_batch import CachedRequestState
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
@@ -250,7 +256,7 @@ class OmniGPUModelRunner(GPUModelRunner):
mrope_pos_ptr += completion_part_len
def _update_states(self, scheduler_output: "SchedulerOutput"):
def _update_states(self, scheduler_output: "SchedulerOutput") -> Callable | None:
"""Update the cached states and the persistent batch with the scheduler
output.
@@ -284,8 +290,7 @@ class OmniGPUModelRunner(GPUModelRunner):
if cleanup_finished_request is not None:
cleanup_finished_request(req_id)
if hasattr(self, "late_interaction_runner"):
self.late_interaction_runner.on_requests_finished(scheduler_output.finished_req_ids)
self.late_interaction_runner.on_requests_finished(scheduler_output.finished_req_ids)
# Remove the finished requests from the persistent batch.
# NOTE(woosuk): There could be an edge case where finished_req_ids and
# scheduled_req_ids overlap. This happens when a request is aborted and
@@ -326,8 +331,9 @@ class OmniGPUModelRunner(GPUModelRunner):
for req_id in unscheduled_req_ids:
self.input_batch.remove_request(req_id)
if self.use_async_spec_decode:
self.prev_num_draft_tokens.np.fill(0)
is_ngram_gpu = self.speculative_config is not None and self.speculative_config.use_ngram_gpu()
if is_ngram_gpu:
ngram_gpu_new_reqs: list[CachedRequestState] = []
reqs_to_add: list[CachedRequestState] = []
deferred_spec_decode_corrections = []
@@ -379,8 +385,7 @@ class OmniGPUModelRunner(GPUModelRunner):
lora_request=new_req_data.lora_request,
)
self.requests[req_id] = req_state
if hasattr(self, "late_interaction_runner"):
self.late_interaction_runner.register_request(req_id, pooling_params)
self.late_interaction_runner.register_request(req_id, pooling_params)
# If prompt embeddings are provided, decode and attach to inter_data
try:
@@ -429,12 +434,30 @@ class OmniGPUModelRunner(GPUModelRunner):
self._init_xdrope_positions(req_state)
reqs_to_add.append(self.requests[req_id])
# Track new requests for ngram_gpu full tensor copy
if is_ngram_gpu:
ngram_gpu_new_reqs.append(req_state)
# Update the states of the running/resumed requests.
is_last_rank = get_pp_group().is_last_rank
req_data = scheduler_output.scheduled_cached_reqs
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
# Save scheduler-allocated spec lengths before trimming so
# prev_num_draft_len keeps the optimistic count for rejection correction.
original_num_spec_per_req: dict[str, int] = {}
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
for req_id, toks in scheduled_spec_tokens.items():
original_num_spec_per_req[req_id] = len(toks)
update_scheduler_for_invalid_drafts(
self._num_valid_draft_tokens_event,
self._num_valid_draft_tokens_cpu,
scheduler_output,
self.input_batch.req_id_to_index,
)
if self.use_async_spec_decode:
self.prev_num_draft_tokens.np.fill(0)
for i, req_id in enumerate(req_data.req_ids):
req_state = self.requests[req_id]
num_computed_tokens = req_data.num_computed_tokens[i]
@@ -473,6 +496,9 @@ class OmniGPUModelRunner(GPUModelRunner):
if prev_req_index is not None:
self.prev_num_draft_tokens.np[prev_req_index] = optimistic_num_accepted
if is_ngram_gpu and optimistic_num_accepted > 0:
self.input_batch.num_tokens_no_spec[req_index] += optimistic_num_accepted
# Update the cached states.
req_state.num_computed_tokens = num_computed_tokens
@@ -521,6 +547,9 @@ class OmniGPUModelRunner(GPUModelRunner):
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
reqs_to_add.append(req_state)
# Track resumed requests for ngram_gpu full tensor copy
if is_ngram_gpu:
ngram_gpu_new_reqs.append(req_state)
continue
# Update the persistent batch.
@@ -539,6 +568,11 @@ class OmniGPUModelRunner(GPUModelRunner):
# Add spec_token_ids to token_ids_cpu.
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)
# Restore scheduler-side draft count after ngram trimming.
if original_num_spec_per_req:
orig = original_num_spec_per_req.get(req_id, 0)
if orig != req_state.prev_num_draft_len:
req_state.prev_num_draft_len = orig
# Add the new or resumed requests to the persistent batch.
# The smaller empty indices are filled first.
@@ -553,6 +587,18 @@ class OmniGPUModelRunner(GPUModelRunner):
# Refresh batch metadata with any pending updates.
self.input_batch.refresh_metadata()
# Incrementally update ngram_gpu tensors after batch is stable
if is_ngram_gpu:
update_ngram_gpu_tensors_incremental(
self.input_batch,
self.token_ids_gpu_tensor,
self.num_tokens_no_spec_gpu,
ngram_gpu_new_reqs,
self.device,
_pinned_idx_buf=self._ngram_pinned_idx_buf,
_pinned_val_buf=self._ngram_pinned_val_buf,
)
if deferred_spec_decode_corrections:
def correct_spec_decode_token_counts():
@@ -577,6 +623,9 @@ class OmniGPUModelRunner(GPUModelRunner):
if cur_req_index is None:
continue
self.input_batch.num_computed_tokens_cpu[cur_req_index] -= correction
if is_ngram_gpu and correction > 0:
self.input_batch.num_tokens_no_spec[cur_req_index] -= correction
self.num_tokens_no_spec_gpu[cur_req_index] -= correction
return correct_spec_decode_token_counts
else:
@@ -846,7 +895,7 @@ class OmniGPUModelRunner(GPUModelRunner):
device=self.device,
)
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens_padded, None, False)
intermediate_tensors = self.sync_and_gather_intermediate_tensors(num_tokens_padded, None, False)
if ubatch_slices_padded is not None:
# Adjust values to reflect a single ubatch.
@@ -900,7 +949,7 @@ class OmniGPUModelRunner(GPUModelRunner):
):
assert isinstance(
self.drafter,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
)
assert self.speculative_config is not None
# Eagle currently only supports PIECEWISE cudagraphs.
@@ -1300,7 +1349,7 @@ class OmniGPUModelRunner(GPUModelRunner):
intermediate_tensors = None
else:
assert intermediate_tensors is not None
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
num_input_tokens, intermediate_tensors, True
)