[Feat] Add MUSA flash attention support via mate package (#2451)
Signed-off-by: Xiaodong Ye <yeahdongcn@gmail.com>
This commit is contained in:
1 parent
c3c736d488
commit
fb3c6bd9b1
5 files changed
+49
-6
No files matched your search
@@ -12,6 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Adapted from https://github.com/huggingface/transformers/blob/main/src/transformers/modeling_flash_attention_utils.py
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
@@ -38,8 +40,10 @@ elif current_omni_platform.is_xpu():
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
elif current_omni_platform.is_musa():
|
||||
# XXX (MUSA): Add MUSA-specific Flash Attention when available
|
||||
pass
|
||||
try:
|
||||
from mate import flash_attn_varlen_func # noqa: F401
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
else:
|
||||
# CUDA: try FA3 -> FA2 fallback chain
|
||||
# Try FA3 from fa3-fwd PyPI package
|
||||
@@ -76,6 +80,12 @@ else:
|
||||
HAS_FLASH_ATTN = flash_attn_func is not None or flash_attn_varlen_func is not None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_mate_available() -> bool:
|
||||
"""Check if MATE (MUSA Flash Attention) is available."""
|
||||
return current_omni_platform.is_musa() and flash_attn_varlen_func is not None
|
||||
|
||||
|
||||
def _index_first_axis(tensor, indices):
|
||||
"""
|
||||
A local implementation of the PyTorch indexing operation `tensor[indices]` on the first axis,
|
||||
|
||||
Reference in new issue
Block a user