[NPU] upgrade to v0.30.0 (#8027)
Signed-off-by: Weiming Liao <liaowm5@gmail.com>
This commit is contained in:
1 parent
d319d276f9
commit
81bc7116c1
6 files changed
+22
-21
No files matched your search
@@ -544,7 +544,7 @@ class NPUARModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_state_copy_funcs(),
|
||||
preprocess_bufs,
|
||||
)
|
||||
# preprocess_mamba resets num_accepted_tokens_cpu to 1
|
||||
|
||||
@@ -269,7 +269,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
self._get_mamba_state_copy_funcs(),
|
||||
preprocess_bufs,
|
||||
)
|
||||
# preprocess_mamba resets num_accepted_tokens_cpu to 1
|
||||
@@ -854,8 +854,6 @@ class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
|
||||
if self.uses_mrope:
|
||||
positions = self.mrope_positions.gpu[:, :num_tokens_padded]
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_tokens_padded]
|
||||
else:
|
||||
positions = self.positions[:num_tokens_padded]
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
@@ -35,9 +36,13 @@ else:
|
||||
|
||||
|
||||
class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
def initialize_kv_cache(self, kv_cache_config) -> None:
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config, kv_cache_allocation_context: AbstractContextManager | None = None
|
||||
) -> None:
|
||||
"""Stage the omni prefix-cache config (hidden / mm tensors reused on hits)."""
|
||||
NPUModelRunner.initialize_kv_cache(self, kv_cache_config)
|
||||
NPUModelRunner.initialize_kv_cache(
|
||||
self, kv_cache_config, kv_cache_allocation_context=kv_cache_allocation_context
|
||||
)
|
||||
if getattr(self, "_omni_prefix_cache_cfg", None) is None:
|
||||
# Same gate as the GPU runner (pooling stage, kv_consumer /
|
||||
# kv_both, hybrid kv groups). Read the config back off
|
||||
@@ -332,8 +337,6 @@ class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):
|
||||
|
||||
if self.uses_mrope:
|
||||
positions = self.mrope_positions.gpu[:, :num_tokens_padded]
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
positions = self.xdrope_positions.gpu[:, :num_tokens_padded]
|
||||
else:
|
||||
positions = self.positions[:num_tokens_padded]
|
||||
|
||||
|
||||
Reference in new issue
Block a user