[Perf] Minimax-H3 support fused RMSNorm and RoPE opt (#5801)

Signed-off-by: Fan <fan@FandeMacBook-Air.local>
Co-authored-by: Fan <fan@FandeMacBook-Air.local>

tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py

@@ -61,6 +61,33 @@ def test_packed_attention_is_a_regional_compile_boundary():
     assert getattr(MiniMaxH3Attention._run_packed_attention, "_torchdynamo_disable", False)
 
 
+def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims():
+    from vllm_omni.diffusion.layers.rope import RotaryEmbedding
+    from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
+        MiniMaxH3Attention,
+    )
+
+    attention = object.__new__(MiniMaxH3Attention)
+    nn.Module.__init__(attention)
+    attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
+    attention.rope._forward_method = attention.rope.forward_native
+
+    x = torch.randn(11, 3, 128, dtype=torch.bfloat16)
+    freqs_half = torch.randn(11, 48)
+    freqs = torch.cat((freqs_half, freqs_half), dim=-1)
+    actual = attention._apply_rope(x, freqs)
+
+    cos = torch.cos(freqs).to(x.dtype).unsqueeze(1)
+    sin = torch.sin(freqs).to(x.dtype).unsqueeze(1)
+    x_rot = x[..., :96]
+    x1, x2 = x_rot.chunk(2, dim=-1)
+    expected_rot = x_rot * cos + torch.cat((-x2, x1), dim=-1) * sin
+    expected = torch.cat((expected_rot, x[..., 96:]), dim=-1)
+
+    torch.testing.assert_close(actual, expected, atol=0, rtol=0)
+    torch.testing.assert_close(actual[..., 96:], x[..., 96:], atol=0, rtol=0)
+
+
 @pytest.mark.parametrize(
     ("tp_size", "message"),
     [