[Rebase] Rebase to vllm v0.21.0 (#3530)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk> Signed-off-by: NumberWan <wantszkin2003@gmail.com> Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com> Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com> Signed-off-by: Dnoob <dxpouo@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Signed-off-by: rein yang <ruiruyang2@gmail.com> Signed-off-by: Nick Cao <ncao@redhat.com> Signed-off-by: zhumingjue <zhumingjue@huawei.com> Signed-off-by: Ricardo Noriega De Soto <rnoriega@redhat.com> Signed-off-by: lyj-jjj <liuyingjun5@huawei.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: weizhoublue <weizhoublue@github.com> Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> Signed-off-by: dengyunyang <584797741@qq.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Signed-off-by: David Chen <530634352@qq.com> Signed-off-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: natureofnature <wzliu@connect.hku.hk> Signed-off-by: bjf-frz <frz123db@gmail.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Signed-off-by: KexiongYu <yukexiong1@huawei.com> Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: NumberWan <wantszkin2003@gmail.com> Co-authored-by: dsinghvi <divyanshsinghvi@gmail.com> Co-authored-by: Dnoob <dxpouo@gmail.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Samit <285365963@qq.com> Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com> Co-authored-by: Nick Cao <ncao@redhat.com> Co-authored-by: zhumingjue138 <zhumingjue@huawei.com> Co-authored-by: Ricardo Noriega <rnoriega@redhat.com> Co-authored-by: lyj-jjj <liuyingjun5@huawei.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: weizhoublue <45163302+weizhoublue@users.noreply.github.com> Co-authored-by: weizhoublue <weizhoublue@github.com> Co-authored-by: dengyunyang <584797741@qq.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Co-authored-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: NATURE <wzliu@connect.hku.hk> Co-authored-by: bjf-frz <frz123db@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Y. Fisher <yukexiong1@huawei.com> Co-authored-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
+20









Zhou Taichang
Claude Opus 4.7
NumberWan
dsinghvi
Dnoob
Yueqian Lin
Hongsheng Liu
knlnguyen1802
Samit
rein yang
Nick Cao
zhumingjue138
Ricardo Noriega
lyj-jjj
Cursor
gcanlin
wangyu
weizhoublue
weizhoublue
dengyunyang
汪志鹏
WeiQing Chen
Jie Liu
Yueqian Lin
NATURE
bjf-frz
amy-why-3459
Y. Fisher
Zeyu Huang | 黃澤宇
Copilot Autofix powered by AI authored and
GitHub committed 2026-05-16 18:18:43 +08:00
1 parent
5e1986206f
commit
eb4e60ee64
46 files changed
+589
-258
No files matched your search
@@ -20,7 +20,9 @@ from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
extract_routed_experts_for_current_batch,
|
||||
get_global_experts_capturer,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
|
||||
@@ -28,6 +30,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker.gpu_model_runner import (
|
||||
@@ -358,11 +361,21 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
self.flush_full_payload_outputs(flush_ids)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
|
||||
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
|
||||
# the modification has influence on the scheduler_output in engine core process.
|
||||
# The replace is much faster than deepcopy.
|
||||
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
|
||||
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
|
||||
spec_decode_tokens_copy = scheduler_output.scheduled_spec_decode_tokens.copy()
|
||||
scheduler_output = replace(
|
||||
scheduler_output,
|
||||
num_scheduled_tokens=num_scheduled_tokens_copy,
|
||||
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
|
||||
)
|
||||
|
||||
if has_kv_transfer_group():
|
||||
kv_connector_metadata = scheduler_output.kv_connector_metadata
|
||||
@@ -678,6 +691,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.routed_experts_initialized and hasattr(self, "_positions_cpu"):
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _sample(
|
||||
@@ -829,7 +850,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
if use_gpu_toks:
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
|
||||
)
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
if input_fits_in_drafter:
|
||||
@@ -1013,12 +1034,15 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
self.accumulate_full_payload_output(rid, pooler_output[i], req_state)
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
|
||||
routed_experts_dict = None
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
output = OmniModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
req_id_to_index=req_id_to_index_output_copy,
|
||||
@@ -1030,6 +1054,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
|
||||
num_nans_in_logits=num_nans_in_logits,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
output.kv_extracted_req_ids = kv_extracted_req_ids
|
||||
output.omni_connector_output = self.get_omni_connector_output()
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
import gc
|
||||
import logging
|
||||
from copy import copy
|
||||
from dataclasses import replace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -18,7 +19,9 @@ from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
extract_routed_experts_for_current_batch,
|
||||
get_global_experts_capturer,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
@@ -27,6 +30,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker.gpu_model_runner import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
@@ -111,11 +115,21 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
self.flush_full_payload_outputs(flush_ids)
|
||||
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
|
||||
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
|
||||
# the modification has influence on the scheduler_output in engine core process.
|
||||
# The replace is much faster than deepcopy.
|
||||
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
|
||||
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
|
||||
spec_decode_tokens_copy = scheduler_output.scheduled_spec_decode_tokens.copy()
|
||||
scheduler_output = replace(
|
||||
scheduler_output,
|
||||
num_scheduled_tokens=num_scheduled_tokens_copy,
|
||||
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
|
||||
)
|
||||
|
||||
if has_kv_transfer_group():
|
||||
kv_connector_metadata = scheduler_output.kv_connector_metadata
|
||||
@@ -339,6 +353,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.routed_experts_initialized and hasattr(self, "_positions_cpu"):
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -425,6 +447,15 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
# [Omni] Copy req_id mappings to avoid async scheduling mutation.
|
||||
req_ids_output_copy = self.input_batch.req_ids.copy()
|
||||
req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy()
|
||||
routed_experts_dict = None
|
||||
if self.routed_experts_initialized:
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
if self._should_accumulate_full_payload_output():
|
||||
for i, rid in enumerate(req_ids_output_copy):
|
||||
req_state = self.requests.get(rid)
|
||||
@@ -442,6 +473,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
num_nans_in_logits={},
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
output.omni_connector_output = self.get_omni_connector_output()
|
||||
|
||||
@@ -751,7 +783,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens_padded, None, False)
|
||||
intermediate_tensors = self.sync_and_gather_intermediate_tensors(num_tokens_padded, None, False)
|
||||
|
||||
if ubatch_slices_padded is not None:
|
||||
# Adjust values to reflect a single ubatch.
|
||||
@@ -794,7 +826,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
|
||||
)
|
||||
assert self.speculative_config is not None
|
||||
# Eagle currently only supports PIECEWISE cudagraphs.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import contextlib
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -18,6 +19,11 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.spec_decode.ngram_proposer_gpu import (
|
||||
update_ngram_gpu_tensors_incremental,
|
||||
update_scheduler_for_invalid_drafts,
|
||||
)
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner, IntermediateTensors, PerLayerAttnMetadata
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
@@ -250,7 +256,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
mrope_pos_ptr += completion_part_len
|
||||
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput"):
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput") -> Callable | None:
|
||||
"""Update the cached states and the persistent batch with the scheduler
|
||||
output.
|
||||
|
||||
@@ -284,8 +290,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if cleanup_finished_request is not None:
|
||||
cleanup_finished_request(req_id)
|
||||
|
||||
if hasattr(self, "late_interaction_runner"):
|
||||
self.late_interaction_runner.on_requests_finished(scheduler_output.finished_req_ids)
|
||||
self.late_interaction_runner.on_requests_finished(scheduler_output.finished_req_ids)
|
||||
# Remove the finished requests from the persistent batch.
|
||||
# NOTE(woosuk): There could be an edge case where finished_req_ids and
|
||||
# scheduled_req_ids overlap. This happens when a request is aborted and
|
||||
@@ -326,8 +331,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
for req_id in unscheduled_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
if self.use_async_spec_decode:
|
||||
self.prev_num_draft_tokens.np.fill(0)
|
||||
is_ngram_gpu = self.speculative_config is not None and self.speculative_config.use_ngram_gpu()
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs: list[CachedRequestState] = []
|
||||
|
||||
reqs_to_add: list[CachedRequestState] = []
|
||||
deferred_spec_decode_corrections = []
|
||||
@@ -379,8 +385,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
lora_request=new_req_data.lora_request,
|
||||
)
|
||||
self.requests[req_id] = req_state
|
||||
if hasattr(self, "late_interaction_runner"):
|
||||
self.late_interaction_runner.register_request(req_id, pooling_params)
|
||||
self.late_interaction_runner.register_request(req_id, pooling_params)
|
||||
|
||||
# If prompt embeddings are provided, decode and attach to inter_data
|
||||
try:
|
||||
@@ -429,12 +434,30 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self._init_xdrope_positions(req_state)
|
||||
|
||||
reqs_to_add.append(self.requests[req_id])
|
||||
# Track new requests for ngram_gpu full tensor copy
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs.append(req_state)
|
||||
|
||||
# Update the states of the running/resumed requests.
|
||||
is_last_rank = get_pp_group().is_last_rank
|
||||
req_data = scheduler_output.scheduled_cached_reqs
|
||||
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
|
||||
# Save scheduler-allocated spec lengths before trimming so
|
||||
# prev_num_draft_len keeps the optimistic count for rejection correction.
|
||||
original_num_spec_per_req: dict[str, int] = {}
|
||||
if self.speculative_config is not None and self.speculative_config.use_ngram_gpu():
|
||||
for req_id, toks in scheduled_spec_tokens.items():
|
||||
original_num_spec_per_req[req_id] = len(toks)
|
||||
update_scheduler_for_invalid_drafts(
|
||||
self._num_valid_draft_tokens_event,
|
||||
self._num_valid_draft_tokens_cpu,
|
||||
scheduler_output,
|
||||
self.input_batch.req_id_to_index,
|
||||
)
|
||||
if self.use_async_spec_decode:
|
||||
self.prev_num_draft_tokens.np.fill(0)
|
||||
|
||||
for i, req_id in enumerate(req_data.req_ids):
|
||||
req_state = self.requests[req_id]
|
||||
num_computed_tokens = req_data.num_computed_tokens[i]
|
||||
@@ -473,6 +496,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if prev_req_index is not None:
|
||||
self.prev_num_draft_tokens.np[prev_req_index] = optimistic_num_accepted
|
||||
|
||||
if is_ngram_gpu and optimistic_num_accepted > 0:
|
||||
self.input_batch.num_tokens_no_spec[req_index] += optimistic_num_accepted
|
||||
|
||||
# Update the cached states.
|
||||
req_state.num_computed_tokens = num_computed_tokens
|
||||
|
||||
@@ -521,6 +547,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
|
||||
|
||||
reqs_to_add.append(req_state)
|
||||
# Track resumed requests for ngram_gpu full tensor copy
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs.append(req_state)
|
||||
continue
|
||||
|
||||
# Update the persistent batch.
|
||||
@@ -539,6 +568,11 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
# Add spec_token_ids to token_ids_cpu.
|
||||
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)
|
||||
# Restore scheduler-side draft count after ngram trimming.
|
||||
if original_num_spec_per_req:
|
||||
orig = original_num_spec_per_req.get(req_id, 0)
|
||||
if orig != req_state.prev_num_draft_len:
|
||||
req_state.prev_num_draft_len = orig
|
||||
|
||||
# Add the new or resumed requests to the persistent batch.
|
||||
# The smaller empty indices are filled first.
|
||||
@@ -553,6 +587,18 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
# Refresh batch metadata with any pending updates.
|
||||
self.input_batch.refresh_metadata()
|
||||
|
||||
# Incrementally update ngram_gpu tensors after batch is stable
|
||||
if is_ngram_gpu:
|
||||
update_ngram_gpu_tensors_incremental(
|
||||
self.input_batch,
|
||||
self.token_ids_gpu_tensor,
|
||||
self.num_tokens_no_spec_gpu,
|
||||
ngram_gpu_new_reqs,
|
||||
self.device,
|
||||
_pinned_idx_buf=self._ngram_pinned_idx_buf,
|
||||
_pinned_val_buf=self._ngram_pinned_val_buf,
|
||||
)
|
||||
|
||||
if deferred_spec_decode_corrections:
|
||||
|
||||
def correct_spec_decode_token_counts():
|
||||
@@ -577,6 +623,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if cur_req_index is None:
|
||||
continue
|
||||
self.input_batch.num_computed_tokens_cpu[cur_req_index] -= correction
|
||||
if is_ngram_gpu and correction > 0:
|
||||
self.input_batch.num_tokens_no_spec[cur_req_index] -= correction
|
||||
self.num_tokens_no_spec_gpu[cur_req_index] -= correction
|
||||
|
||||
return correct_spec_decode_token_counts
|
||||
else:
|
||||
@@ -846,7 +895,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(num_tokens_padded, None, False)
|
||||
intermediate_tensors = self.sync_and_gather_intermediate_tensors(num_tokens_padded, None, False)
|
||||
|
||||
if ubatch_slices_padded is not None:
|
||||
# Adjust values to reflect a single ubatch.
|
||||
@@ -900,7 +949,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | ExtractHiddenStatesProposer | Gemma4Proposer,
|
||||
)
|
||||
assert self.speculative_config is not None
|
||||
# Eagle currently only supports PIECEWISE cudagraphs.
|
||||
@@ -1300,7 +1349,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
intermediate_tensors = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
intermediate_tensors = self.sync_and_slice_intermediate_tensors(
|
||||
intermediate_tensors = self.sync_and_gather_intermediate_tensors(
|
||||
num_input_tokens, intermediate_tensors, True
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user