rebase: adapt to vLLM v0.29 APIs
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
6a63f5cb99
commit
87a1fd8da2
60 files changed
+2038
-514
No files matched your search
@@ -1,3 +1,4 @@
|
||||
from importlib import import_module
|
||||
from importlib.util import find_spec
|
||||
|
||||
import torch
|
||||
@@ -98,6 +99,7 @@ class RotaryEmbedding(CustomOp):
|
||||
self.interleaved = not is_neox_style
|
||||
self.half_head_dim = half_head_dim
|
||||
self.apply_rotary_emb_flash_attn = None
|
||||
self.apply_rotary_emb_vllm_flash_attn = None
|
||||
self.has_mindie = False
|
||||
# ``find_spec("flash_attn")`` is True as long as *any* package publishes
|
||||
# the ``flash_attn`` namespace — including ``flash-attn-4``, which ships
|
||||
@@ -111,6 +113,17 @@ class RotaryEmbedding(CustomOp):
|
||||
self.apply_rotary_emb_flash_attn = apply_rotary
|
||||
except ImportError:
|
||||
pass
|
||||
# CUDA wheels vendor this kernel under vllm_flash_attn, while source
|
||||
# and CPU-only installations may not contain the generated package.
|
||||
# Resolve it lazily and retain the native implementation as a portable
|
||||
# fallback for those environments.
|
||||
if current_omni_platform.is_cuda():
|
||||
try:
|
||||
self.apply_rotary_emb_vllm_flash_attn = import_module(
|
||||
"vllm.vllm_flash_attn.layers.rotary"
|
||||
).apply_rotary_emb
|
||||
except ImportError:
|
||||
pass
|
||||
if find_spec("mindiesd") is not None:
|
||||
self.has_mindie = True
|
||||
|
||||
@@ -135,12 +148,18 @@ class RotaryEmbedding(CustomOp):
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb
|
||||
|
||||
cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)
|
||||
|
||||
if self.apply_rotary_emb_vllm_flash_attn is None:
|
||||
return apply_rotary_emb_torch(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
interleaved=self.interleaved,
|
||||
)
|
||||
|
||||
x, squeezed = _ensure_batch_dim(x)
|
||||
output = apply_rotary_emb(
|
||||
output = self.apply_rotary_emb_vllm_flash_attn(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
@@ -231,13 +250,14 @@ class RotaryEmbeddingWan(RotaryEmbedding):
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb
|
||||
if self.apply_rotary_emb_vllm_flash_attn is None:
|
||||
return self.forward_native(x, cos, sin)
|
||||
|
||||
if cos.dim() > 2:
|
||||
cos = cos.reshape(-1, cos.shape[-1])
|
||||
sin = sin.reshape(-1, sin.shape[-1])
|
||||
|
||||
return apply_rotary_emb(
|
||||
return self.apply_rotary_emb_vllm_flash_attn(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
|
||||
Reference in new issue
Block a user