[Refactor] Unify Snake/SnakeBeta and alias-free activation into common modules (#3886)

Signed-off-by: BeatSeat <wendavid552@gmail.com>
This commit is contained in:
BeatSeat authored and GitHub committed 2026-05-27 23:41:35 -05:00
1 parent 2b999429c4
commit dab596e468
6 files changed
+320 -616

No files matched your search

@@ -0,0 +1,243 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Alias-free activation for BigVGAN-style speech decoders.
Provides anti-aliased activation (upsample → activation → downsample) with
Kaiser-windowed sinc filters. Includes NPU bf16 workarounds for platforms
where F.pad(mode='replicate') does not support bfloat16.
Used by: Qwen2.5-Omni, Qwen3-TTS v1, CoVo-Audio, and future BigVGAN vocoders.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
# ---------------------------------------------------------------------------
# Helper utilities
# ---------------------------------------------------------------------------
def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) -> torch.Tensor:
"""Generate a 1-D Kaiser-windowed sinc low-pass filter.
Args:
cutoff: Normalized cutoff frequency (0 to 0.5).
half_width: Transition bandwidth.
kernel_size: Number of filter taps.
Returns:
Filter tensor of shape ``(1, 1, kernel_size)``.
"""
is_even = kernel_size % 2 == 0
half_size = kernel_size // 2
delta_f = 4 * half_width
attenuation = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
if attenuation > 50.0:
beta = 0.1102 * (attenuation - 8.7)
elif attenuation >= 21.0:
beta = 0.5842 * (attenuation - 21) ** 0.4 + 0.07886 * (attenuation - 21.0)
else:
beta = 0.0
# torch.kaiser_window is not supported on NPU / XPU — compute on CPU.
if current_omni_platform.is_npu():
kaiser_window = torch.kaiser_window(
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
).to("npu")
elif current_omni_platform.is_xpu():
kaiser_window = torch.kaiser_window(
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
).to("xpu")
else:
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
if is_even:
time_indices = torch.arange(-half_size, half_size) + 0.5
else:
time_indices = torch.arange(kernel_size) - half_size
if cutoff == 0:
return torch.zeros((1, 1, kernel_size), dtype=torch.float32)
sinc_filter = torch.sinc(2 * cutoff * time_indices)
normalized_filter = 2 * cutoff * kaiser_window * sinc_filter
normalized_filter /= normalized_filter.sum()
return normalized_filter.view(1, 1, kernel_size)
def replication_pad_1d(hidden_states: torch.Tensor, pad_left: int, pad_right: int) -> torch.Tensor:
"""Manual replicate padding for NPU bf16 compatibility.
``F.pad(mode='replicate')`` does not support bfloat16 on NPU (and older
CUDA). This expand+cat fallback produces identical results.
TODO: Remove when upstream PyTorch fixes replication_pad1d for bf16.
"""
if pad_left == 0 and pad_right == 0:
return hidden_states
segments = []
if pad_left > 0:
left = hidden_states[..., :1].expand(*hidden_states.shape[:-1], pad_left)
segments.append(left)
segments.append(hidden_states)
if pad_right > 0:
right = hidden_states[..., -1:].expand(*hidden_states.shape[:-1], pad_right)
segments.append(right)
return torch.cat(segments, dim=-1)
# ---------------------------------------------------------------------------
# Up / Down sampling with Kaiser sinc filters
# ---------------------------------------------------------------------------
class UpSample1d(nn.Module):
"""Kaiser-sinc interpolation upsample (default: 2x, 12-tap filter)."""
def __init__(self, ratio: int = 2, kernel_size: int | None = None):
super().__init__()
self.ratio = ratio
self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
self.stride = ratio
self.pad = self.kernel_size // ratio - 1
self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
filt = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size)
self.register_buffer("filter", filt, persistent=False)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
channels = hidden_states.shape[1]
if current_omni_platform.is_npu():
input_dtype = hidden_states.dtype
hidden_states = replication_pad_1d(hidden_states.to(self.filter.dtype), self.pad, self.pad)
hidden_states = self.ratio * F.conv_transpose1d(
hidden_states,
self.filter.expand(channels, -1, -1),
stride=self.stride,
groups=channels,
).to(input_dtype)
else:
input_dtype = hidden_states.dtype
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(self.filter.dtype)
hidden_states = self.ratio * F.conv_transpose1d(
hidden_states,
self.filter.expand(channels, -1, -1),
stride=self.stride,
groups=channels,
).to(input_dtype)
hidden_states = hidden_states[..., self.pad_left : -self.pad_right]
return hidden_states
class DownSample1d(nn.Module):
"""Kaiser-sinc anti-aliasing downsample (default: 2x, 12-tap filter)."""
def __init__(self, ratio: int = 2, kernel_size: int | None = None):
super().__init__()
if kernel_size is None:
kernel_size = int(6 * ratio // 2) * 2
cutoff = 0.5 / ratio
half_width = 0.6 / ratio
if cutoff < 0.0:
raise ValueError("Minimum cutoff must be larger than zero.")
if cutoff > 0.5:
raise ValueError("A cutoff above 0.5 does not make sense.")
self.even = kernel_size % 2 == 0
self.pad_left = kernel_size // 2 - int(self.even)
self.pad_right = kernel_size // 2
self.stride = ratio
filt = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)
self.register_buffer("filter", filt, persistent=False)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
channels = hidden_states.shape[1]
if current_omni_platform.is_npu():
input_dtype = hidden_states.dtype
hidden_states = replication_pad_1d(hidden_states.to(self.filter.dtype), self.pad_left, self.pad_right)
out = F.conv1d(
hidden_states,
self.filter.to(device=hidden_states.device, dtype=hidden_states.dtype).expand(channels, -1, -1),
stride=self.stride,
groups=channels,
).to(input_dtype)
else:
input_dtype = hidden_states.dtype
hidden_states = F.pad(hidden_states, (self.pad_left, self.pad_right), mode="replicate").to(
self.filter.dtype
)
out = F.conv1d(
hidden_states,
self.filter.expand(channels, -1, -1),
stride=self.stride,
groups=channels,
).to(input_dtype)
return out
# ---------------------------------------------------------------------------
# Alias-free activation: upsample → activation → downsample
# ---------------------------------------------------------------------------
class AliasFreeActivation1d(nn.Module):
"""Anti-aliased activation following BigVGAN (ICLR 2023).
Upsamples the signal, applies a pointwise activation (typically SnakeBeta),
then downsamples with a low-pass filter to suppress aliasing from the
high-frequency components introduced by the activation function.
The ``activation`` should be an ``nn.Module`` instance — typically
``SnakeBeta`` from ``common.snake_activation``, which already dispatches
between Triton and eager PyTorch internally.
Args:
activation: Pointwise activation module (e.g. ``SnakeBeta(channels)``).
up_ratio: Upsample factor before activation.
down_ratio: Downsample factor after activation.
up_kernel_size: Kaiser sinc filter taps for upsampling.
down_kernel_size: Kaiser sinc filter taps for downsampling.
"""
def __init__(
self,
activation: nn.Module,
up_ratio: int = 2,
down_ratio: int = 2,
up_kernel_size: int = 12,
down_kernel_size: int = 12,
):
super().__init__()
if not callable(activation):
raise TypeError("Activation function must be callable")
self.act = activation
self.upsample = UpSample1d(up_ratio, up_kernel_size)
self.downsample = DownSample1d(down_ratio, down_kernel_size)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = self.upsample(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states = self.downsample(hidden_states)
return hidden_states
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The Qwen team.
"""Shared SnakeBeta activation for speech decoders (Qwen3-TTS, Qwen3-Omni Code2Wav)."""
"""Shared Snake/SnakeBeta activations for speech decoders.
Used by: Qwen3-TTS, Qwen3-Omni Code2Wav, Qwen2.5-Omni, CoVo-Audio, CosyVoice3.
"""
import torch
from torch import nn
@@ -70,12 +73,17 @@ class SnakeBeta(nn.Module):
SnakeBeta._triton_kernel = _kernel
return True
def __init__(self, in_features, alpha=1.0):
def __init__(self, in_features, alpha=1.0, alpha_logscale=True):
super().__init__()
self.in_features = in_features
self.alpha_logscale = alpha_logscale
self.alpha = Parameter(torch.zeros(in_features) * alpha)
self.beta = Parameter(torch.zeros(in_features) * alpha)
if alpha_logscale:
self.alpha = Parameter(torch.zeros(in_features) * alpha)
self.beta = Parameter(torch.zeros(in_features) * alpha)
else:
self.alpha = Parameter(torch.ones(in_features) * alpha)
self.beta = Parameter(torch.ones(in_features) * alpha)
self.no_div_by_zero = 0.000000001
@@ -86,8 +94,12 @@ class SnakeBeta(nn.Module):
def precompute_exp_cache(self):
"""Materialize exp(alpha) and 1/(exp(beta)+eps) as frozen buffers."""
with torch.no_grad():
self._exp_alpha = torch.exp(self.alpha).contiguous()
self._inv_beta = (1.0 / (torch.exp(self.beta) + self.no_div_by_zero)).contiguous()
if self.alpha_logscale:
self._exp_alpha = torch.exp(self.alpha).contiguous()
self._inv_beta = (1.0 / (torch.exp(self.beta) + self.no_div_by_zero)).contiguous()
else:
self._exp_alpha = self.alpha.contiguous()
self._inv_beta = (1.0 / (self.beta + self.no_div_by_zero)).contiguous()
@property
def _cached(self):
@@ -104,12 +116,10 @@ class SnakeBeta(nn.Module):
return self._eager_forward(hidden_states)
def _eager_forward(self, hidden_states):
if self._cached:
exp_alpha = self._exp_alpha.unsqueeze(0).unsqueeze(-1)
inv_beta = self._inv_beta.unsqueeze(0).unsqueeze(-1)
else:
exp_alpha = torch.exp(self.alpha).unsqueeze(0).unsqueeze(-1)
inv_beta = (1.0 / (torch.exp(self.beta) + self.no_div_by_zero)).unsqueeze(0).unsqueeze(-1)
if not self._cached:
self.precompute_exp_cache()
exp_alpha = self._exp_alpha.unsqueeze(0).unsqueeze(-1)
inv_beta = self._inv_beta.unsqueeze(0).unsqueeze(-1)
hidden_states = hidden_states + inv_beta * torch.pow(torch.sin(hidden_states * exp_alpha), 2)
return hidden_states
@@ -134,3 +144,33 @@ class SnakeBeta(nn.Module):
block_t=block_t,
)
return out
class Snake(SnakeBeta):
"""Original Snake activation with a single parameter: x + 1/α * sin²(αx).
Unlike SnakeBeta which has separate alpha (frequency) and beta (magnitude)
parameters, Snake uses alpha for both. Only ``alpha`` appears in the
state_dict — ``beta`` is absent, keeping checkpoint compatibility with
CosyVoice3's HiFi-GAN.
The Triton kernel and precomputed-cache path from SnakeBeta are reused;
``precompute_exp_cache`` derives ``_inv_beta`` from ``alpha`` so the
forward path is identical.
"""
def __init__(self, in_features, alpha=1.0, alpha_logscale=False):
# Initialise SnakeBeta — creates both alpha and beta Parameters.
super().__init__(in_features, alpha=alpha, alpha_logscale=alpha_logscale)
# Drop beta as a Parameter so it won't appear in state_dict.
del self.beta
def precompute_exp_cache(self):
"""Derive both exp_alpha and inv_beta from the single alpha parameter."""
with torch.no_grad():
if self.alpha_logscale:
self._exp_alpha = torch.exp(self.alpha).contiguous()
self._inv_beta = (1.0 / (torch.exp(self.alpha) + self.no_div_by_zero)).contiguous()
else:
self._exp_alpha = self.alpha.contiguous()
self._inv_beta = (1.0 / (self.alpha + self.no_div_by_zero)).contiguous()