(Phase 1)Add ModelOpt FP8 auto-detect support for diffusion checkpoints #2709 (#2913)

Signed-off-by: roG0d <rodgarcas98@gmail.com>
Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
Co-authored-by: roG0d <rodgarcas98@gmail.com>
This commit is contained in:
baonudesifeizhaiandroG0d authored and GitHub committed 2026-05-09 16:55:05 +08:00
1 parent 40a07e0d80
commit c4a0990044
26 files changed
+1214 -176

No files matched your search

+22 -2
View File
@@ -65,6 +65,22 @@ def apply_rotary_emb_mindiesd(
return rotary_position_embedding(x, cos, sin, rotated_mode="rotated_half", head_first=False, fused=True)
def _ensure_batch_dim(x: torch.Tensor) -> tuple[torch.Tensor, bool]:
# Upstream fused rotary kernels expect ``x`` shaped as
# ``[batch_size, seq_len, nheads, headdim]``. Some omni diffusion call
# sites pass ``[seq_len, nheads, headdim]`` instead, so normalize to 4D
# here before entering the fused path.
if x.dim() == 3:
return x.unsqueeze(0), True
return x, False
def _restore_batch_dim(x: torch.Tensor, squeezed: bool) -> torch.Tensor:
if squeezed:
return x.squeeze(0)
return x
class RotaryEmbedding(CustomOp):
"""
rotary positional embedding.
@@ -98,12 +114,14 @@ class RotaryEmbedding(CustomOp):
cos = cos[0]
sin = sin[0]
return apply_rotary_emb(
x, squeezed = _ensure_batch_dim(x)
output = apply_rotary_emb(
x,
cos,
sin,
interleaved=self.interleaved,
)
return _restore_batch_dim(output, squeezed)
def forward_hip(
self,
@@ -119,12 +137,14 @@ class RotaryEmbedding(CustomOp):
cos = cos[0]
sin = sin[0]
return self.apply_rotary_emb_flash_attn(
x, squeezed = _ensure_batch_dim(x)
output = self.apply_rotary_emb_flash_attn(
x,
cos,
sin,
interleaved=self.interleaved,
)
return _restore_batch_dim(output, squeezed)
def forward_npu(
self,