[Rebase] Rebase to vllm v0.21.0 (#3530)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk>
Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com>
Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com>
Signed-off-by: Dnoob <dxpouo@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Signed-off-by: rein yang <ruiruyang2@gmail.com>
Signed-off-by: Nick Cao <ncao@redhat.com>
Signed-off-by: zhumingjue <zhumingjue@huawei.com>
Signed-off-by: Ricardo Noriega De Soto <rnoriega@redhat.com>
Signed-off-by: lyj-jjj <liuyingjun5@huawei.com>
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: weizhoublue <weizhoublue@github.com>
Signed-off-by: weizhou.lan@daocloud.io <weizhou.lan@daocloud.io>
Signed-off-by: dengyunyang <584797741@qq.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Signed-off-by: David Chen <530634352@qq.com>
Signed-off-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
Signed-off-by: bjf-frz <frz123db@gmail.com>
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Signed-off-by: KexiongYu <yukexiong1@huawei.com>
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: NumberWan <wantszkin2003@gmail.com>
Co-authored-by: dsinghvi <divyanshsinghvi@gmail.com>
Co-authored-by: Dnoob <dxpouo@gmail.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Co-authored-by: Samit <285365963@qq.com>
Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com>
Co-authored-by: Nick Cao <ncao@redhat.com>
Co-authored-by: zhumingjue138 <zhumingjue@huawei.com>
Co-authored-by: Ricardo Noriega <rnoriega@redhat.com>
Co-authored-by: lyj-jjj <liuyingjun5@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Co-authored-by: weizhoublue <45163302+weizhoublue@users.noreply.github.com>
Co-authored-by: weizhoublue <weizhoublue@github.com>
Co-authored-by: dengyunyang <584797741@qq.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
Co-authored-by: Jie Liu <33612777+keeper-jie@users.noreply.github.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: NATURE <wzliu@connect.hku.hk>
Co-authored-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Y. Fisher <yukexiong1@huawei.com>
Co-authored-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
46 files changed
+589 -258

No files matched your search

@@ -16,7 +16,11 @@ from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
from vllm.forward_context import BatchDescriptor
from vllm.logger import logger
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import RoutedExpertsCapturer
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
extract_routed_experts_for_current_batch,
get_global_experts_capturer,
issue_routing_d2h_copy,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import (
@@ -28,8 +32,8 @@ from vllm.v1.outputs import (
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.structured_output.utils import apply_grammar_bitmask
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker import mamba_utils
from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata
from vllm.v1.worker.mamba_utils import preprocess_mamba
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
from vllm_ascend.ascend_forward_context import set_ascend_forward_context
from vllm_ascend.attention.utils import AscendCommonAttentionMetadata
@@ -246,11 +250,9 @@ class NPUARModelRunner(OmniNPUModelRunner):
intermediate_tensors: IntermediateTensors | None = None,
) -> OmniModelRunnerOutput | IntermediateTensors | None:
if self.vllm_config.model_config.enable_return_routed_experts:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.clear_buffer()
else:
logger.warning("RoutedExpertsCapturer is not initialized.")
capturer = get_global_experts_capturer()
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
capturer.finalize_pending_copy()
if self.ascend_config.profiling_chunk_config.enabled:
self._sync_device()
self._execution_start_time = time.perf_counter()
@@ -299,7 +301,9 @@ class NPUARModelRunner(OmniNPUModelRunner):
# TODO(Ronald1995): deepcopy is expensive when there is a large
# number of requests, optimize it later.
if ((
self.use_async_scheduling and self.num_spec_tokens and self._draft_token_ids is None # type: ignore[has-type]
self.use_async_scheduling
and self.num_spec_tokens
and self._draft_token_ids is None # type: ignore[has-type]
) or (
# NOTE: This branch specifically triggers a deepcopy during the prefill phase
# only for PCP (Parallel Context Processing) + Multi-Modal (MM) scenarios.
@@ -455,7 +459,7 @@ class NPUARModelRunner(OmniNPUModelRunner):
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
deferred_state_corrections_fn = None
mamba_utils.preprocess_mamba(
preprocess_mamba(
scheduler_output,
self.kv_cache_config,
self.cache_config,
@@ -715,6 +719,15 @@ class NPUARModelRunner(OmniNPUModelRunner):
# previous model forward without breaking async scheduling.
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
return None
@torch.inference_mode()
@@ -858,12 +871,17 @@ class NPUARModelRunner(OmniNPUModelRunner):
if self.speculative_config is not None:
self.finalize_kv_connector()
routed_experts_dict = None
if self.model_config.enable_return_routed_experts:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.save_captured_experts(indices=self.cpu_slot_mapping)
else:
logger.warning("RoutedExpertsCapturer is not initialized.")
capturer = get_global_experts_capturer()
if capturer is not None and hasattr(self.input_batch, "num_tokens_no_spec"):
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
# -------------------------------------- Omni-new -------------------------------------------------
engine_output_type, downstream_req_ids = self._resolve_pooler_payload_req_ids(req_ids_output_copy)
@@ -984,6 +1002,7 @@ class NPUARModelRunner(OmniNPUModelRunner):
kv_connector_output=kv_connector_output,
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
cudagraph_stats=cudagraph_stats,
routed_experts_dict=routed_experts_dict,
)
model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids
# -------------------------------------- Omni-new -------------------------------------------------
@@ -15,14 +15,18 @@ from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import logger
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import RoutedExpertsCapturer
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
extract_routed_experts_for_current_batch,
get_global_experts_capturer,
issue_routing_d2h_copy,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output
from vllm.v1.utils import record_function_or_nullcontext
from vllm.v1.worker import mamba_utils
from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput, PerLayerAttnMetadata
from vllm.v1.worker.mamba_utils import preprocess_mamba
from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
from vllm_ascend.ascend_forward_context import set_ascend_forward_context
@@ -74,11 +78,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
intermediate_tensors: IntermediateTensors | None = None,
) -> OmniModelRunnerOutput | IntermediateTensors | None:
if self.vllm_config.model_config.enable_return_routed_experts:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.clear_buffer()
else:
logger.warning("RoutedExpertsCapturer is not initialized.")
capturer = get_global_experts_capturer()
if capturer is not None and hasattr(capturer, "finalize_pending_copy"):
capturer.finalize_pending_copy()
if self.ascend_config.profiling_chunk_config.enabled:
self._sync_device()
self._execution_start_time = time.perf_counter()
@@ -227,7 +229,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
deferred_state_corrections_fn = None
mamba_utils.preprocess_mamba(
preprocess_mamba(
scheduler_output,
self.kv_cache_config,
self.cache_config,
@@ -383,6 +385,15 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
# previous model forward without breaking async scheduling.
if deferred_state_corrections_fn:
deferred_state_corrections_fn()
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(self, "_positions_cpu"):
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
return None
@torch.inference_mode()
@@ -468,6 +479,17 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
# [Omni] Copy req_id mappings to avoid async scheduling mutation.
req_ids_output_copy = self.input_batch.req_ids.copy()
req_id_to_index_output_copy = self.input_batch.req_id_to_index.copy()
routed_experts_dict = None
if self.vllm_config.model_config.enable_return_routed_experts and hasattr(
self.input_batch, "num_tokens_no_spec"
):
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
output = OmniModelRunnerOutput(
req_ids=req_ids_output_copy,
req_id_to_index=req_id_to_index_output_copy,
@@ -478,6 +500,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner):
kv_connector_output=kv_connector_output,
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
cudagraph_stats=cudagraph_stats,
routed_experts_dict=routed_experts_dict,
)
# -------------------------------------- Omni-new -------------------------------------------------