[Perf] Fuse MiniMax H3 SwiGLU activation (#6283)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
07ceb3324a
commit
9683b7f4b4
2 files changed
+40
-2
No files matched your search
@@ -33,6 +33,7 @@ from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
SequenceParallelOutput,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.activation import SiluAndMul
|
||||
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
@@ -586,6 +587,7 @@ class MiniMaxH3MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc1",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
# Chunk the fused fc1 output as [gate, up], then compute
|
||||
# silu(gate) * up.
|
||||
self.fc2 = RowParallelLinear(
|
||||
@@ -600,8 +602,7 @@ class MiniMaxH3MLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
hidden, _ = self.fc1(x)
|
||||
gate, up = hidden.chunk(2, dim=-1)
|
||||
hidden = nn.functional.silu(gate) * up
|
||||
hidden = self.act_fn(hidden)
|
||||
out, _ = self.fc2(hidden)
|
||||
return out
|
||||
|
||||
|
||||
Reference in new issue
Block a user