[Feature] support sp for hunyuan (#2163)
Signed-off-by: dengyunyang <584797741@qq.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
1 parent
c139ee4aed
commit
c0d90b64c6
6 files changed
+346
-27
No files matched your search
@@ -373,8 +373,14 @@ class UlyssesParallelAttention:
|
||||
orig_head_cnt=int(orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||
joint_orig_head_cnt=int(joint_orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||
)
|
||||
|
||||
use_2d_mask = False
|
||||
if attn_metadata is not None:
|
||||
if attn_metadata.attn_mask is not None and attn_metadata.attn_mask.ndim == 2:
|
||||
use_2d_mask = True
|
||||
if attn_metadata.joint_attn_mask is not None and attn_metadata.joint_attn_mask.ndim == 2:
|
||||
use_2d_mask = True
|
||||
|
||||
if attn_metadata is not None and use_2d_mask:
|
||||
if is_joint:
|
||||
if attn_metadata.joint_attn_mask is None and attn_metadata.attn_mask is None:
|
||||
attn_metadata.attn_mask = None
|
||||
|
||||
Reference in new issue
Block a user