diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index 05a780542..eaeef693c 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.26.0 +ARG VLLM_ASCEND_TAG=v0.27.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.a3 b/docker/Dockerfile.npu.a3 index d226a505f..2dc6334d9 100644 --- a/docker/Dockerfile.npu.a3 +++ b/docker/Dockerfile.npu.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.26.0-a3 +ARG VLLM_ASCEND_TAG=v0.27.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} # WORKDIR /vllm-workspace/vllm diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index 848a52f24..f5af5fb31 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.26.0 +ARG VLLM_ASCEND_TAG=v0.27.0 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index 7ff709ee8..0529cb2da 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -1,5 +1,5 @@ ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend -ARG VLLM_ASCEND_TAG=v0.26.0-a3 +ARG VLLM_ASCEND_TAG=v0.27.0-a3 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG} ARG APP_DIR=/vllm-workspace/vllm-omni diff --git a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 0ff7144a9..63f3912f7 100644 --- a/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -730,7 +730,9 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): has_encoder_input = self.model_config.is_encoder_decoder and num_encoder_reqs > 0 # Run forward pass - clear_kv_metadata = self.speculative_config is None + defer_kv_connector_finalize = self.speculative_config is not None and ( + get_pp_group().is_last_rank or self.broadcast_pp_output + ) with ( record_function_or_nullcontext("forward"), set_ascend_forward_context( @@ -744,13 +746,12 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin): model_instance=self.model, skip_compiled=has_encoder_input, has_sinks=self._has_sinks, - input_ids=input_ids, eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ), self.maybe_get_kv_connector_output( scheduler_output, **( - {"defer_finalize": not clear_kv_metadata} + {"defer_finalize": defer_kv_connector_finalize} ), ) as kv_connector_output, ): diff --git a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index b728b3cc3..94c6f8916 100644 --- a/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -381,7 +381,9 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin has_encoder_input = self.model_config.is_encoder_decoder and num_encoder_reqs > 0 # Run forward pass - clear_kv_metadata = self.speculative_config is None + defer_kv_connector_finalize = self.speculative_config is not None and ( + get_pp_group().is_last_rank or self.broadcast_pp_output + ) with ( record_function_or_nullcontext("forward"), set_ascend_forward_context( @@ -395,12 +397,11 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin model_instance=self.model, skip_compiled=has_encoder_input, has_sinks=self._has_sinks, - input_ids=input_ids, eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ), self.maybe_get_kv_connector_output( scheduler_output, - **({"defer_finalize": not clear_kv_metadata}), + **({"defer_finalize": defer_kv_connector_finalize}), ) as kv_connector_output, ): if self.cache_config.mamba_cache_mode == "align": @@ -654,9 +655,10 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin num_active_loras: int = 0, profile_seq_lens: int | None = None, profile_cpp: bool = False, + randomize_inputs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: # only support eager mode and piecewise graph now - assert cudagraph_runtime_mode is None or cudagraph_runtime_mode.valid_runtime_modes() + assert cudagraph_runtime_mode is None or cudagraph_runtime_mode.is_valid_runtime_mode() # If cudagraph_mode.decode_mode() == FULL and # cudagraph_mode.separate_routine(). This means that we are using # different graphs and/or modes for mixed prefill-decode batches vs. @@ -917,7 +919,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin batch_descriptor=batch_desc, model_instance=self.model, has_sinks=self._has_sinks, - input_ids=input_ids, eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, ): outputs = self.model( diff --git a/vllm_omni/platforms/npu/worker/npu_model_runner.py b/vllm_omni/platforms/npu/worker/npu_model_runner.py index bc8e4f0a2..375d36047 100644 --- a/vllm_omni/platforms/npu/worker/npu_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_model_runner.py @@ -125,9 +125,10 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): num_active_loras: int = 0, profile_seq_lens: int | None = None, profile_cpp: bool = False, + randomize_inputs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: # only support eager mode and piecewise graph now - assert cudagraph_runtime_mode is None or cudagraph_runtime_mode.valid_runtime_modes() + assert cudagraph_runtime_mode is None or cudagraph_runtime_mode.is_valid_runtime_mode() # If cudagraph_mode.decode_mode() == FULL and # cudagraph_mode.separate_routine(). This means that we are using # different graphs and/or modes for mixed prefill-decode batches vs. @@ -367,19 +368,21 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner): if hasattr(self.drafter, "model") and hasattr(self.drafter.model, "compute_logits"): return self.drafter.model.compute_logits(hidden_states[dummy_indices]) - with set_ascend_forward_context( - attn_metadata, - self.vllm_config, - num_tokens=num_tokens_padded, - num_tokens_across_dp=num_tokens_across_dp, - in_profile_run=is_profile, - num_actual_tokens=num_tokens_padded, - aclgraph_runtime_mode=cudagraph_runtime_mode, - batch_descriptor=batch_desc, - model_instance=self.model, - has_sinks=self._has_sinks, - input_ids=input_ids, - eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, + with ( + self.maybe_randomize_inputs(input_ids, inputs_embeds, randomize_inputs=randomize_inputs), + set_ascend_forward_context( + attn_metadata, + self.vllm_config, + num_tokens=num_tokens_padded, + num_tokens_across_dp=num_tokens_across_dp, + in_profile_run=is_profile, + num_actual_tokens=num_tokens_padded, + aclgraph_runtime_mode=cudagraph_runtime_mode, + batch_descriptor=batch_desc, + model_instance=self.model, + has_sinks=self._has_sinks, + eplb_heat_collection_status=self.eplb_heat_collection_status if self.dynamic_eplb else False, + ), ): # ---------------------------------------Omni-new---------------------------------------------- if getattr(self.model, "talker", None) is not None and self.has_talker_mtp: