[Bugfix/Feature] Remove Hardcoded Flash Attention in Bagel & Support GQA in SDPA Backend (#3728)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
593c4c06aa
commit
68a1d23e4d
5 files changed
+151
-451
No files matched your search
@@ -85,6 +85,7 @@ class SDPAImpl(AttentionImpl):
|
||||
) -> None:
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
self.requires_gqa = num_heads != num_kv_heads
|
||||
if backend_kwargs:
|
||||
logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))
|
||||
|
||||
@@ -111,6 +112,7 @@ class SDPAImpl(AttentionImpl):
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
enable_gqa=self.requires_gqa,
|
||||
)
|
||||
out = output.permute(0, 2, 1, 3)
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user