[Rebase] Rebase to vllm v0.21.0 (#3530)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk> Signed-off-by: NumberWan <wantszkin2003@gmail.com> Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com> Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com> Signed-off-by: Dnoob <dxpouo@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Signed-off-by: rein yang <ruiruyang2@gmail.com> Signed-off-by: Nick Cao <ncao@redhat.com> Signed-off-by: zhumingjue <zhumingjue@huawei.com> Signed-off-by: Ricardo Noriega De Soto <rnoriega@redhat.com> Signed-off-by: lyj-jjj <liuyingjun5@huawei.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: weizhoublue <weizhoublue@github.com> Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io> Signed-off-by: dengyunyang <584797741@qq.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Signed-off-by: David Chen <530634352@qq.com> Signed-off-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: natureofnature <wzliu@connect.hku.hk> Signed-off-by: bjf-frz <frz123db@gmail.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Signed-off-by: KexiongYu <yukexiong1@huawei.com> Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: NumberWan <wantszkin2003@gmail.com> Co-authored-by: dsinghvi <divyanshsinghvi@gmail.com> Co-authored-by: Dnoob <dxpouo@gmail.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Samit <285365963@qq.com> Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com> Co-authored-by: Nick Cao <ncao@redhat.com> Co-authored-by: zhumingjue138 <zhumingjue@huawei.com> Co-authored-by: Ricardo Noriega <rnoriega@redhat.com> Co-authored-by: lyj-jjj <liuyingjun5@huawei.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: weizhoublue <45163302+weizhoublue@users.noreply.github.com> Co-authored-by: weizhoublue <weizhoublue@github.com> Co-authored-by: dengyunyang <584797741@qq.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Co-authored-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: NATURE <wzliu@connect.hku.hk> Co-authored-by: bjf-frz <frz123db@gmail.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Y. Fisher <yukexiong1@huawei.com> Co-authored-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
+20









Zhou Taichang
Claude Opus 4.7
NumberWan
dsinghvi
Dnoob
Yueqian Lin
Hongsheng Liu
knlnguyen1802
Samit
rein yang
Nick Cao
zhumingjue138
Ricardo Noriega
lyj-jjj
Cursor
gcanlin
wangyu
weizhoublue
weizhoublue
dengyunyang
汪志鹏
WeiQing Chen
Jie Liu
Yueqian Lin
NATURE
bjf-frz
amy-why-3459
Y. Fisher
Zeyu Huang | 黃澤宇
Copilot Autofix powered by AI authored and
GitHub committed 2026-05-16 18:18:43 +08:00
1 parent
5e1986206f
commit
eb4e60ee64
46 files changed
+589
-258
No files matched your search
@@ -16,7 +16,11 @@ from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
from vllm.forward_context import BatchDescriptor
|
||||
from vllm.logger import logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import RoutedExpertsCapturer
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
extract_routed_experts_for_current_batch,
|
||||
get_global_experts_capturer,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import (
|
||||
@@ -28,8 +32,8 @@ from vllm.v1.outputs import (
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker import mamba_utils
|
||||
from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata
|
||||
from vllm.v1.worker.mamba_utils import preprocess_mamba
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
from vllm_ascend.ascend_forward_context import set_ascend_forward_context
|
||||
from vllm_ascend.attention.utils import AscendCommonAttentionMetadata
|
||||
@@ -246,11 +250,9 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> OmniModelRunnerOutput | IntermediateTensors | None:
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer()
|
||||
else:
|
||||
logger.warning("RoutedExpertsCapturer is not initialized.")
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
if self.ascend_config.profiling_chunk_config.enabled:
|
||||
self._sync_device()
|
||||
self._execution_start_time = time.perf_counter()
|
||||
@@ -299,7 +301,9 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
# TODO(Ronald1995): deepcopy is expensive when there is a large
|
||||
# number of requests, optimize it later.
|
||||
if ((
|
||||
self.use_async_scheduling and self.num_spec_tokens and self._draft_token_ids is None # type: ignore[has-type]
|
||||
self.use_async_scheduling
|
||||
and self.num_spec_tokens
|
||||
and self._draft_token_ids is None # type: ignore[has-type]
|
||||
) or (
|
||||
# NOTE: This branch specifically triggers a deepcopy during the prefill phase
|
||||
# only for PCP (Parallel Context Processing) + Multi-Modal (MM) scenarios.
|
||||
@@ -455,7 +459,7 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
deferred_state_corrections_fn = None
|
||||
mamba_utils.preprocess_mamba(
|
||||
preprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.cache_config,
|
||||
@@ -715,6 +719,15 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
# previous model forward without breaking async scheduling.
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -858,12 +871,17 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
if self.speculative_config is not None:
|
||||
self.finalize_kv_connector()
|
||||
|
||||
routed_experts_dict = None
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.save_captured_experts(indices=self.cpu_slot_mapping)
|
||||
else:
|
||||
logger.warning("RoutedExpertsCapturer is not initialized.")
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None and hasattr(self.input_batch, "num_tokens_no_spec"):
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
engine_output_type, downstream_req_ids = self._resolve_pooler_payload_req_ids(req_ids_output_copy)
|
||||
@@ -984,6 +1002,7 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
kv_connector_output=kv_connector_output,
|
||||
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
|
||||
@@ -15,14 +15,18 @@ from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
|
||||
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import RoutedExpertsCapturer
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
extract_routed_experts_for_current_batch,
|
||||
get_global_experts_capturer,
|
||||
issue_routing_d2h_copy,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm.v1.worker import mamba_utils
|
||||
from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata
|
||||
from vllm.v1.worker.mamba_utils import preprocess_mamba
|
||||
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
|
||||
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
|
||||
from vllm_ascend.ascend_forward_context import set_ascend_forward_context
|
||||
@@ -74,11 +78,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> OmniModelRunnerOutput | IntermediateTensors | None:
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer()
|
||||
else:
|
||||
logger.warning("RoutedExpertsCapturer is not initialized.")
|
||||
capturer = get_global_experts_capturer()
|
||||
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
|
||||
capturer.finalize_pending_copy()
|
||||
if self.ascend_config.profiling_chunk_config.enabled:
|
||||
self._sync_device()
|
||||
self._execution_start_time = time.perf_counter()
|
||||
@@ -227,7 +229,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
deferred_state_corrections_fn = None
|
||||
mamba_utils.preprocess_mamba(
|
||||
preprocess_mamba(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.cache_config,
|
||||
@@ -383,6 +385,15 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
|
||||
# previous model forward without breaking async scheduling.
|
||||
if deferred_state_corrections_fn:
|
||||
deferred_state_corrections_fn()
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
|
||||
issue_routing_d2h_copy(
|
||||
input_batch_req_ids=self.input_batch.req_ids,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
positions=self.positions,
|
||||
positions_cpu=self._positions_cpu,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -468,6 +479,17 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
|
||||
# [Omni] Copy req_id mappings to avoid async scheduling mutation.
|
||||
req_ids_output_copy = self.input_batch.req_ids.copy()
|
||||
req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy()
|
||||
routed_experts_dict = None
|
||||
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(
|
||||
self.input_batch, "num_tokens_no_spec"
|
||||
):
|
||||
routed_experts_dict = extract_routed_experts_for_current_batch(
|
||||
req_ids=req_ids_output_copy,
|
||||
requests=self.requests,
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
|
||||
max_model_len=self.max_model_len,
|
||||
)
|
||||
output = OmniModelRunnerOutput(
|
||||
req_ids=req_ids_output_copy,
|
||||
req_id_to_index=req_id_to_index_output_copy,
|
||||
@@ -478,6 +500,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
|
||||
kv_connector_output=kv_connector_output,
|
||||
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
routed_experts_dict=routed_experts_dict,
|
||||
)
|
||||
# -------------------------------------- Omni-new -------------------------------------------------
|
||||
|
||||
|
||||
Reference in new issue
Block a user