Signed-off-by: roG0d <rodgarcas98@gmail.com> Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com> Co-authored-by: roG0d <rodgarcas98@gmail.com>
This commit is contained in:
1 parent
40a07e0d80
commit
c4a0990044
26 files changed
+1214
-176
No files matched your search
@@ -65,6 +65,22 @@ def apply_rotary_emb_mindiesd(
|
||||
return rotary_position_embedding(x, cos, sin, rotated_mode="rotated_half", head_first=False, fused=True)
|
||||
|
||||
|
||||
def _ensure_batch_dim(x: torch.Tensor) -> tuple[torch.Tensor, bool]:
|
||||
# Upstream fused rotary kernels expect ``x`` shaped as
|
||||
# ``[batch_size, seq_len, nheads, headdim]``. Some omni diffusion call
|
||||
# sites pass ``[seq_len, nheads, headdim]`` instead, so normalize to 4D
|
||||
# here before entering the fused path.
|
||||
if x.dim() == 3:
|
||||
return x.unsqueeze(0), True
|
||||
return x, False
|
||||
|
||||
|
||||
def _restore_batch_dim(x: torch.Tensor, squeezed: bool) -> torch.Tensor:
|
||||
if squeezed:
|
||||
return x.squeeze(0)
|
||||
return x
|
||||
|
||||
|
||||
class RotaryEmbedding(CustomOp):
|
||||
"""
|
||||
rotary positional embedding.
|
||||
@@ -98,12 +114,14 @@ class RotaryEmbedding(CustomOp):
|
||||
cos = cos[0]
|
||||
sin = sin[0]
|
||||
|
||||
return apply_rotary_emb(
|
||||
x, squeezed = _ensure_batch_dim(x)
|
||||
output = apply_rotary_emb(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
interleaved=self.interleaved,
|
||||
)
|
||||
return _restore_batch_dim(output, squeezed)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
@@ -119,12 +137,14 @@ class RotaryEmbedding(CustomOp):
|
||||
cos = cos[0]
|
||||
sin = sin[0]
|
||||
|
||||
return self.apply_rotary_emb_flash_attn(
|
||||
x, squeezed = _ensure_batch_dim(x)
|
||||
output = self.apply_rotary_emb_flash_attn(
|
||||
x,
|
||||
cos,
|
||||
sin,
|
||||
interleaved=self.interleaved,
|
||||
)
|
||||
return _restore_batch_dim(output, squeezed)
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
|
||||
Reference in new issue
Block a user