[Perf] MiniMax H3 Qwen3-VL support fused RMSNorm on NPU (#5915)
Signed-off-by: Wenjia Li <wjialish@gmail.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
7cd826d28c
commit
38a2500bdf
2 files changed
+49
-15
No files matched your search
@@ -39,6 +39,8 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm as DiffusionRMSNorm
|
||||
|
||||
MINIMAX_H3_QWEN3VL_SELECTED_LM_LAYER = 50
|
||||
MINIMAX_H3_QWEN3VL_HIDDEN_DIM = 5120
|
||||
|
||||
@@ -291,18 +293,22 @@ class MiniMaxH3Qwen3VLRowParallelLinear(nn.Module):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MiniMaxH3Qwen3VLRMSNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
class MiniMaxH3Qwen3VLRMSNorm(DiffusionRMSNorm):
|
||||
"""Qwen3-VL RMSNorm using the common implementation.
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
return self.weight * hidden_states.to(input_dtype)
|
||||
The model-specific name keeps checkpoint weight keys stable, while the
|
||||
common RMSNorm dispatches to torch_npu.npu_rms_norm on Ascend. Gamma uses
|
||||
the checkpoint/model dtype, but native fallbacks keep the RMS variance
|
||||
reduction and scaling in float32.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> None:
|
||||
super().__init__(hidden_size, eps=eps, dtype=dtype)
|
||||
|
||||
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -685,8 +691,8 @@ class MiniMaxH3Qwen3VLTextAttention(nn.Module):
|
||||
dtype=dtype,
|
||||
input_is_parallel=True,
|
||||
)
|
||||
self.q_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.q_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype)
|
||||
self.k_norm = MiniMaxH3Qwen3VLRMSNorm(self.head_dim, eps=config.rms_norm_eps, dtype=dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -760,8 +766,10 @@ class MiniMaxH3Qwen3VLTextDecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.self_attn = MiniMaxH3Qwen3VLTextAttention(group, config, dtype)
|
||||
self.mlp = MiniMaxH3Qwen3VLTextMLP(group, config, dtype)
|
||||
self.input_layernorm = MiniMaxH3Qwen3VLRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = MiniMaxH3Qwen3VLRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.input_layernorm = MiniMaxH3Qwen3VLRMSNorm(config.hidden_size, eps=config.rms_norm_eps, dtype=dtype)
|
||||
self.post_attention_layernorm = MiniMaxH3Qwen3VLRMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps, dtype=dtype
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in new issue
Block a user