[Bugfix][Fish Speech] Flatten Fast AR RoPE positions (#3299)
Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
e403acda3f
commit
368afea2f2
1 file changed
+1
-1
@@ -106,7 +106,7 @@ class _FastARAttention(nn.Module):
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
|
||||
|
||||
q, k = self.rotary_emb(position_ids, q, k)
|
||||
q, k = self.rotary_emb(position_ids.reshape(-1), q, k)
|
||||
|
||||
q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
|
||||
k = k.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
|
||||
|
||||
Reference in new issue
Block a user