[Bugfix] Register LTX RMSNorm identity weight as buffer (#4278)
Signed-off-by: mglyn <1203789601@qq.com>
This commit is contained in:
1 parent
cb1c5a0c39
commit
02eb7a4ecb
2 files changed
+33
-1
No files matched your search
@@ -62,7 +62,17 @@ def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) ->
|
||||
kwargs["has_weight"] = elementwise_affine
|
||||
elif not elementwise_affine:
|
||||
raise TypeError("RMSNorm backend does not support disabling affine weights.")
|
||||
return RMSNorm(hidden_size, **kwargs)
|
||||
norm = RMSNorm(hidden_size, **kwargs)
|
||||
weight = getattr(norm, "weight", None)
|
||||
if (
|
||||
not elementwise_affine
|
||||
and isinstance(weight, torch.Tensor)
|
||||
and not isinstance(weight, nn.Parameter)
|
||||
and "weight" not in dict(norm.named_buffers())
|
||||
):
|
||||
delattr(norm, "weight")
|
||||
norm.register_buffer("weight", weight, persistent=False)
|
||||
return norm
|
||||
|
||||
|
||||
def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
|
||||
|
||||
Reference in new issue
Block a user