[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-09-26 22:51:04 +08:00
1 parent 8a49c65bf9
commit e1303e5067
55 files changed
+4222 -230

No files matched your search

@@ -1077,6 +1077,64 @@ class CodePredictorWrapper(nn.Module):
]
)
def _predict_step_logits(
self,
proj_buf: torch.Tensor,
bsz: int,
padded_bsz: int,
step: int,
is_npu_capturing: bool,
) -> torch.Tensor:
"""Run one residual-codebook step; model wrappers may specialize it."""
device = proj_buf.device
max_seq = self._num_groups + 1
model_fwd = self._compiled_model_fwd
lm_heads = self._lm_heads_list
graph_key: int | tuple[int, int] = padded_bsz
seq_len = max_seq
if self._prefix_reprefill_enabled:
actual_seq_len = step + 1
for prefix_seq_len in self._prefix_reprefill_seq_lens:
if prefix_seq_len < actual_seq_len:
continue
prefix_key = (padded_bsz, prefix_seq_len)
if prefix_key in self._bucket_pos_ids:
graph_key = prefix_key
seq_len = prefix_seq_len
break
pos_ids = self._bucket_pos_ids.get(graph_key)
if pos_ids is None:
pos_ids = (
torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
)
# Use captured device graph if available, otherwise call compiled fn.
device_graph_entry = self._device_graphs.get(graph_key)
# Let the outer graph record the regular forward during capture;
# normal inference still uses the inner graph replay fast path.
if device_graph_entry is not None and not is_npu_capturing:
device_graph_entry[0].replay()
hidden_out = device_graph_entry[1]
else:
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
return lm_heads[step - 1](hidden_out[:bsz, step, :])
def _sample_per_call(
self,
logits: torch.Tensor,
inv_temperature: float,
top_k: int,
generator: _GeneratorLike,
uniforms: torch.Tensor | None,
) -> torch.Tensor:
scaled = logits * inv_temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
return self._sample_codes_gumbel(scaled, generator=generator, uniforms=uniforms)
@torch.inference_mode()
def forward(
self,
@@ -1114,10 +1172,7 @@ class CodePredictorWrapper(nn.Module):
self._ensure_buffers(device, dtype, padded_bsz)
proj_buf = self._proj_buf
max_seq = num_groups + 1
projection = self.small_to_mtp_projection
model_fwd = self._compiled_model_fwd
lm_heads = self._lm_heads_list
codec_embeds = self._codec_embeds_list
# torch-npu cannot replay an inner NPUGraph while the outer talker_mtp
# graph is being captured. Capture state is constant for this forward.
@@ -1153,39 +1208,7 @@ class CodePredictorWrapper(nn.Module):
# Autoregressive loop: predict layers 1..G-1
for step in range(1, num_groups):
graph_key: int | tuple[int, int] = padded_bsz
seq_len = max_seq
if self._prefix_reprefill_enabled:
actual_seq_len = step + 1
for prefix_seq_len in self._prefix_reprefill_seq_lens:
if prefix_seq_len < actual_seq_len:
continue
prefix_key = (padded_bsz, prefix_seq_len)
if prefix_key in self._bucket_pos_ids:
graph_key = prefix_key
seq_len = prefix_seq_len
break
pos_ids = self._bucket_pos_ids.get(graph_key)
if pos_ids is None:
pos_ids = (
torch.arange(seq_len, device=device, dtype=torch.long)
.unsqueeze(0)
.expand(padded_bsz, -1)
.contiguous()
)
# Use captured device graph if available, otherwise call compiled fn.
device_graph_entry = self._device_graphs.get(graph_key)
# Let the outer graph record the regular forward during capture;
# normal inference still uses the inner graph replay fast path.
if device_graph_entry is not None and not is_npu_capturing:
device_graph_entry[0].replay()
hidden_out = device_graph_entry[1]
else:
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
logits = self._predict_step_logits(proj_buf, bsz, padded_bsz, step, is_npu_capturing)
# Sample next code via Gumbel-max.
#
@@ -1216,12 +1239,8 @@ class CodePredictorWrapper(nn.Module):
else:
# "per_call" mode: temperature-scaled + top-k -> Gumbel-max
if use_sampling:
scaled = logits * inv_temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
step_uniforms = sample_uniforms[:, step - 1, :] if sample_uniforms is not None else None
code = self._sample_codes_gumbel(scaled, generator=sample_generator, uniforms=step_uniforms)
code = self._sample_per_call(logits, inv_temperature, top_k, sample_generator, step_uniforms)
else:
code = logits.argmax(dim=-1, keepdim=True)
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Frame-local KV reuse for the Qwen3-TTS residual codebooks.
The scratch buffer belongs to one predictor worker, just like its projection
buffer. Calls are serialized on the worker's CUDA stream. Every frame starts
by overwriting positions 0 and 1; later queries can only read that frame's
written positions. No request state survives between frames.
"""
from collections.abc import Sequence
import torch
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
CodePredictorAttention,
CodePredictorBaseModel,
CodePredictorDecoderLayer,
_rotate_half,
)
from .short_kv_attention import short_kv_attention
def _attention_cached(
attn: CodePredictorAttention,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
k_cache: torch.Tensor,
v_cache: torch.Tensor,
write_positions: torch.Tensor,
) -> torch.Tensor:
"""Attend only the new positions, reusing earlier positions' keys and values.
``k_cache``/``v_cache`` are ``[B, num_kv_heads, max_seq, head_dim]``;
the new positions' keys and values are written at ``write_positions``
(``[n]``). The attention kernel masks keys after each query position,
so unwritten slots are never read.
"""
bsz, seq_len, _ = hidden_states.shape
qkv = attn.qkv_proj(hidden_states)
q_raw, k_raw, v_raw = attn._split_qkv(qkv)
q = attn.q_norm(q_raw.view(bsz, seq_len, attn.num_heads, attn.head_dim)).transpose(1, 2)
k = attn.k_norm(k_raw.view(bsz, seq_len, attn.num_kv_heads, attn.head_dim)).transpose(1, 2)
v = v_raw.view(bsz, seq_len, attn.num_kv_heads, attn.head_dim).transpose(1, 2)
cos, sin = position_embeddings
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
q = (q * cos) + (_rotate_half(q) * sin)
k = (k * cos) + (_rotate_half(k) * sin)
k_cache.index_copy_(2, write_positions, k.to(k_cache.dtype))
v_cache.index_copy_(2, write_positions, v.to(v_cache.dtype))
attn_out = short_kv_attention(q, k_cache, v_cache, write_positions, attn.scaling)
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
return attn.o_proj(attn_out)
def _layer_cached(
layer: CodePredictorDecoderLayer,
hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
k_cache: torch.Tensor,
v_cache: torch.Tensor,
write_positions: torch.Tensor,
) -> torch.Tensor:
residual = hidden_states
hidden_states = layer.input_layernorm(hidden_states)
hidden_states = _attention_cached(
layer.self_attn, hidden_states, position_embeddings, k_cache, v_cache, write_positions
)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = layer.post_attention_layernorm(hidden_states)
hidden_states = layer.mlp(hidden_states)
return residual + hidden_states
def forward_cached(
model: CodePredictorBaseModel,
inputs_embeds: torch.Tensor,
position_ids: torch.Tensor,
kv_caches: Sequence[tuple[torch.Tensor, torch.Tensor]],
write_positions: torch.Tensor,
) -> torch.Tensor:
"""``forward`` for the new positions only (``position_ids`` ``[B, n]``), reusing earlier K/V."""
hidden_states = inputs_embeds
with torch.amp.autocast(inputs_embeds.device.type, enabled=False):
position_embeddings = model.rotary_emb(hidden_states, position_ids)
for layer, (k_cache, v_cache) in zip(model.layers, kv_caches):
hidden_states = _layer_cached(layer, hidden_states, position_embeddings, k_cache, v_cache, write_positions)
hidden_states = model.norm(hidden_states)
return hidden_states
class FrameLocalKVCache:
"""Non-module execution state; checkpoint parameters remain on the predictor."""
def __init__(self, model: CodePredictorBaseModel, max_batch: int) -> None:
self.model = model
weight = next(model.parameters())
attn = model.layers[0].self_attn
self.buffer = torch.zeros(
len(model.layers),
2,
max_batch,
attn.num_kv_heads,
attn.max_seq,
attn.head_dim,
device=weight.device,
dtype=weight.dtype,
)
self.positions = {
step: torch.arange(0 if step == 1 else step, step + 1, device=weight.device)
for step in range(1, attn.max_seq - 1)
}
# Batch buckets share compiled kernels; outer MTP graphs still capture
# fixed addresses. This helper never creates nested CUDA graphs.
self.forward = torch.compile(forward_cached, dynamic=True, options={"epilogue_fusion": False})
def __call__(self, proj_buf: torch.Tensor, batch: int, step: int) -> torch.Tensor:
if batch > self.buffer.shape[2]:
raise ValueError("Qwen3-TTS cached predictor batch exceeds its configured capacity")
first = 0 if step == 1 else step
caches = [(layer[0, :batch], layer[1, :batch]) for layer in self.buffer]
positions = self.positions[step]
return self.forward(
self.model,
proj_buf[:batch, first : step + 1],
positions.unsqueeze(0).expand(batch, -1),
caches,
positions,
)
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Top-k Gumbel sampling with the outer MTP graph's supplied uniforms."""
import torch
from vllm.triton_utils import tl, tldevice, triton
@triton.jit
def _sample(
logits,
uniforms,
output,
ls: tl.constexpr,
us: tl.constexpr,
lc: tl.constexpr,
uc: tl.constexpr,
vocab: tl.constexpr,
top_k: tl.constexpr,
topk_block: tl.constexpr,
inv_temperature: tl.constexpr,
block: tl.constexpr,
):
row = tl.program_id(0)
idx = tl.arange(0, block)
value = tl.load(logits + row * ls + idx * lc, idx < vocab, other=-float("inf"))
scaled = (value.to(tl.float32) * inv_temperature).to(value.dtype).to(tl.float32)
if top_k > 0:
if top_k == 1:
threshold = tl.max(scaled, 0)
else:
ordered = tl.topk(scaled, topk_block)
threshold = tl.sum(tl.where(tl.arange(0, topk_block) == top_k - 1, ordered, 0))
scaled = tl.where(scaled < threshold, -float("inf"), scaled)
u = tl.load(uniforms + row * us + idx * uc, idx < vocab, other=0.5)
score = scaled - tldevice.log(-tldevice.log(u))
score = tl.where(idx < vocab, score, -float("inf"))
max_score = tl.max(score, 0)
chosen = tl.min(tl.where(score == max_score, idx, block), 0)
first_nan = tl.min(tl.where(score != score, idx, block), 0)
tl.store(output + row, tl.where(first_nan < block, first_nan, chosen))
def sample_code_topk_gumbel(
logits: torch.Tensor, uniforms: torch.Tensor, top_k: int, inv_temperature: float
) -> torch.Tensor:
"""Match BF16 scale rounding, top-k ties, and first-index Gumbel argmax.
Random values are supplied by the outer graph, so RNG state and seeded
per-request streams are unchanged. CUDA libdevice logs match ATen's logs.
"""
if logits.shape != uniforms.shape:
raise ValueError("sampling uniforms must match logits shape")
if logits.device != uniforms.device or not logits.is_cuda:
raise ValueError("logits and uniforms must be on the same CUDA device")
if top_k > logits.shape[-1]:
raise ValueError("top_k must not exceed the vocabulary size")
b, v = logits.shape
out = torch.empty((b, 1), device=logits.device, dtype=torch.int64)
_sample[(b,)](
logits,
uniforms,
out,
logits.stride(0),
uniforms.stride(0),
logits.stride(1),
uniforms.stride(1),
v,
top_k,
triton.next_power_of_2(max(1, top_k)),
inv_temperature,
triton.next_power_of_2(v),
num_warps=4,
)
return out
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-TTS first-frame decode eligibility."""
from typing import Any
import torch
from vllm_omni.platforms import current_omni_platform
def talker_first_audio_enabled(vllm_config: Any) -> bool:
"""First-frame delivery requires CUDA MRv2 streaming with an in-process Talker.
Unsupported runners retain the regular codec path. Prefix-cache replay
and distributed Talkers likewise retain the regular path.
"""
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM as Predictor
extra = Predictor._stage_connector_extra_config(vllm_config)
model = vllm_config.model_config
parallel = vllm_config.parallel_config
return (
Predictor._parse_bool_config(extra.get("talker_first_audio"))
and current_omni_platform.is_cuda()
and torch.device(vllm_config.device_config.device).type == "cuda"
and bool(getattr(model, "use_v2_model_runner", False))
and bool(getattr(model, "async_chunk", False))
and parallel.tensor_parallel_size == 1
and parallel.pipeline_parallel_size == 1
and parallel.distributed_executor_backend in (None, "uni")
and not vllm_config.cache_config.enable_prefix_caching
)
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""First codec frame -> PCM inside the Talker process.
A stream's first frame has no decoder context, so its waveform does not depend
on the Code2Wav stage's streaming state. Decoding it where the frame is
produced removes the stage hop from time to first audio. Code2Wav advances
its causal state without emitting that delivered frame again. Only streams
without reference-code context use this path.
The decode runs beside the Talker's next step, so its latency is set by how
many dependent kernels it issues. A single frame with no context attends to
itself only: each softmax is over one key and equals 1 exactly, so attention
returns the value projection. The single-frame path therefore
skips q/k, RoPE, masks and softmax (about two thirds of the kernels).
"""
from __future__ import annotations
from collections.abc import Sequence
import torch
import torch.nn as nn
from torch.cuda import CUDAGraph
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Decoder
logger = init_logger(__name__)
_SUBFOLDER = "speech_tokenizer"
class Qwen3TTSFirstFrameDecoder(nn.Module):
def __init__(self, model_path: str) -> None:
super().__init__()
self.model_path = model_path
config = Qwen3TTSTokenizerV2Config.from_pretrained(model_path, subfolder=_SUBFOLDER)
self.decoder = Qwen3TTSTokenizerV2Decoder._from_config(config.decoder_config)
self.decoder.eval()
self.num_quantizers = int(config.decoder_config.num_quantizers)
self.sample_rate = int(config.output_sample_rate)
self._graphs: dict[int, tuple[CUDAGraph, torch.Tensor, torch.Tensor]] = {}
def load(self, vllm_config: VllmConfig) -> set[str]:
"""Load decoder weights exactly as Code2Wav does; returns the loaded parameter names."""
loader = DefaultModelLoader(vllm_config.load_config)
source = DefaultModelLoader.Source(
model_or_path=self.model_path,
revision=vllm_config.model_config.revision,
subfolder=_SUBFOLDER,
)
loaded = AutoWeightsLoader(self).load_weights(
loader._get_weights_iterator(source),
mapper=WeightsMapper(orig_to_new_prefix={"encoder.": None}),
)
self.decoder.to(device=vllm_config.device_config.device, dtype=vllm_config.model_config.dtype)
if hasattr(self.decoder, "precompute_snake_caches"):
self.decoder.precompute_snake_caches()
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM as Predictor
extra = Predictor._stage_connector_extra_config(vllm_config)
if Predictor._parse_bool_config(extra.get("decode_time_major_conv")):
self.decoder.enable_time_major_conv()
return loaded
def _decode_single_frame(self, codes: torch.Tensor) -> torch.Tensor:
"""``codes`` [n, num_quantizers, 1] -> waveform; equals ``_decode_xvec_first_chunk``."""
d = self.decoder
hidden = d.pre_conv(d.quantizer.decode(codes)).transpose(1, 2)
t = d.pre_transformer
h = t.input_proj(hidden)
for layer in t.layers[: t.config.num_hidden_layers]:
attention = layer.self_attn
values = attention.v_proj(layer.input_layernorm(h))
if attention.num_key_value_groups != 1:
values = values.reshape(*values.shape[:-1], -1, attention.head_dim)
values = values.repeat_interleave(attention.num_key_value_groups, dim=-2).flatten(-2)
h = h + layer.self_attn_layer_scale(attention.o_proj(values))
h = h + layer.mlp_layer_scale(layer.mlp(layer.post_attention_layernorm(h)))
return d._conv_decode(t.output_proj(t.norm(h))).clamp(min=-1, max=1)
@torch.inference_mode()
def capture(self, batch_sizes: Sequence[int] = (1, 2, 4, 8)) -> None:
device = next(self.decoder.parameters()).device
for batch_size in sorted(set(batch_sizes)):
static_input = torch.zeros(batch_size, self.num_quantizers, 1, dtype=torch.long, device=device)
self._decode_single_frame(static_input)
torch.accelerator.synchronize(device)
graph = CUDAGraph()
# A separate pool prevents overlap with Talker/MTP graph allocations.
with torch.cuda.graph(graph, pool=torch.cuda.graph_pool_handle()):
static_output = self._decode_single_frame(static_input)
self._graphs[batch_size] = (graph, static_input, static_output)
logger.info("Captured Talker first-frame decoder graphs for batch sizes %s", sorted(self._graphs))
@torch.inference_mode()
def decode(self, codes: torch.Tensor) -> torch.Tensor:
"""``codes`` [n, num_quantizers] long on device -> float32 PCM [n, samples] (new tensor)."""
if not self._graphs:
return self._decode_single_frame(codes.unsqueeze(-1))[:, 0, :].float()
outputs = []
largest = max(self._graphs)
for start in range(0, int(codes.shape[0]), largest):
chunk = codes[start : start + largest]
rows = int(chunk.shape[0])
batch_size = min(size for size in self._graphs if size >= rows)
graph, static_input, static_output = self._graphs[batch_size]
static_input.zero_()
static_input[:rows, :, 0].copy_(chunk)
graph.replay()
outputs.append(static_output[:rows, 0, :].to(dtype=torch.float32, copy=True))
return torch.cat(outputs, dim=0)
@@ -6,6 +6,7 @@ from __future__ import annotations
import os
from collections import Counter
from collections.abc import Iterable
from contextlib import nullcontext
from typing import Any
import torch
@@ -16,6 +17,7 @@ from vllm.logger import init_logger
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from vllm_omni.data_entry_keys import FIRST_AUDIO_REQUIRED_KEY
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import parse_chunk_ramp
@@ -112,6 +114,7 @@ class Qwen3TTSCode2Wav(nn.Module):
dec_config = tok_config.decoder_config
self.decoder = Qwen3TTSTokenizerV2Decoder._from_config(dec_config)
self.decoder.eval()
self._num_quantizers = int(dec_config.num_quantizers)
self._output_sample_rate = int(tok_config.output_sample_rate)
self._total_upsample = int(self.decoder.total_upsample)
@@ -302,6 +305,7 @@ class Qwen3TTSCode2Wav(nn.Module):
valid_indices: list[int] = []
left_context_size = [0] * len(request_ids_list)
ref_context_size = [0] * len(request_ids_list)
first_audio_flags = [False] * len(request_ids_list)
segment_finished_flags = [False] * len(request_ids_list)
request_state_ids: list[str | None] = [None] * len(request_ids_list)
ref_context_request_ids: list[str | None] = [None] * len(request_ids_list)
@@ -336,6 +340,7 @@ class Qwen3TTSCode2Wav(nn.Module):
if not isinstance(info, dict):
continue
meta = info.get("meta", {})
first_audio_flags[i] = _meta_bool(meta.get("first_audio", False))
if "is_segment_finished" in meta:
segment_finished_flags[i] = _meta_bool(meta["is_segment_finished"])
if "left_context_size" in meta:
@@ -404,6 +409,7 @@ class Qwen3TTSCode2Wav(nn.Module):
)
state = {}
self._decoder_state_cache[state_req_id] = state
state.setdefault("skip_first_audio", first_audio_flags[i])
state.setdefault("prefix_frames", 0)
if state_req_id.startswith(_DUMMY_REQUEST_ID):
state["_is_dummy_run"] = True
@@ -515,6 +521,10 @@ class Qwen3TTSCode2Wav(nn.Module):
if self._batch_stats_log_every > 0 and self._batch_stats_forwards % self._batch_stats_log_every == 0:
self.log_decode_batch_stats()
first_audio_required = [torch.tensor(False)] * num_req
if request_states is not None:
for row, idx in enumerate(valid_indices):
first_audio_required[idx] = torch.tensor(bool(request_states[row].get("skip_first_audio", False)))
audios: list[torch.Tensor] = [empty] * num_req
srs = [sr_tensor] * num_req
@@ -538,7 +548,7 @@ class Qwen3TTSCode2Wav(nn.Module):
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": srs},
multimodal_outputs={"model_outputs": audios, "sr": srs, FIRST_AUDIO_REQUIRED_KEY: first_audio_required},
)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
@@ -714,6 +724,11 @@ class Qwen3TTSCode2Wav(nn.Module):
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config decode_batch_max_size={decode_batch_max_size}")
self._decode_batch_max_size = decode_batch_max_size
decode_enable_tf32 = _get_bool_config("decode_enable_tf32", False)
decode_time_major_conv = _get_bool_config("decode_time_major_conv", False)
self.decoder.capture_first_audio_state_only = self._async_chunk and _get_bool_config(
"talker_first_audio", False
)
decode_cudnn_benchmark = _get_bool_config("decode_cudnn_benchmark", False)
else:
codec_chunk_frames = 0
codec_left_context_frames = 0
@@ -722,6 +737,8 @@ class Qwen3TTSCode2Wav(nn.Module):
decode_cudagraph_batch_sizes = None
decode_cudagraph_capture_sizes = None
decode_enable_tf32 = False
decode_cudnn_benchmark = False
decode_time_major_conv = False
if decode_enable_tf32 and device.type == "cuda":
# PyTorch exposes TF32 controls as process-wide CUDA backend
@@ -738,21 +755,39 @@ class Qwen3TTSCode2Wav(nn.Module):
torch.get_float32_matmul_precision(),
)
if decode_time_major_conv and device.type == "cuda":
self.decoder.enable_time_major_conv()
self.decoder._initial_codec_chunk_frames = initial_codec_chunk_frames
self.decoder._incremental_chunk_frames = codec_chunk_frames or 25
self.decoder._incremental_chunk_ramp = list(codec_chunk_ramp or ())
if hasattr(self.decoder, "enable_cudagraph") and device.type == "cuda":
try:
self._maybe_enable_decoder_cudagraph(
device=device,
codec_chunk_frames=codec_chunk_frames,
codec_left_context_frames=codec_left_context_frames,
initial_codec_chunk_frames=initial_codec_chunk_frames,
codec_chunk_ramp=codec_chunk_ramp,
decode_cudagraph_batch_sizes=decode_cudagraph_batch_sizes,
decode_cudagraph_capture_sizes=decode_cudagraph_capture_sizes,
# Autotune only during warmup/capture, then restore the process
# flags. The captured convolution algorithms remain in the
# graphs without changing later models' cuDNN policy.
autotune = (
torch.backends.cudnn.flags(
enabled=torch.backends.cudnn.enabled,
benchmark=True,
benchmark_limit=10,
deterministic=torch.backends.cudnn.deterministic,
allow_tf32=torch.backends.cudnn.allow_tf32,
)
if decode_cudnn_benchmark
else nullcontext()
)
with autotune:
self._maybe_enable_decoder_cudagraph(
device=device,
codec_chunk_frames=codec_chunk_frames,
codec_left_context_frames=codec_left_context_frames,
initial_codec_chunk_frames=initial_codec_chunk_frames,
codec_chunk_ramp=codec_chunk_ramp,
decode_cudagraph_batch_sizes=decode_cudagraph_batch_sizes,
decode_cudagraph_capture_sizes=decode_cudagraph_capture_sizes,
)
except Exception:
logger.warning(
"Failed to enable CUDA Graph for Code2Wav decoder",
@@ -1,12 +1,17 @@
"""Qwen3-TTS Code Predictor -- thin wrapper over CodePredictorWrapper."""
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-TTS predictor: per-call sampling and opt-in CUDA frame-local KV reuse."""
from __future__ import annotations
from collections.abc import Iterable
from collections.abc import Iterable, Sequence
from typing import TYPE_CHECKING
import torch
from vllm.config import VllmConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.logger import init_logger
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
CodePredictorBaseModel,
@@ -17,6 +22,11 @@ from vllm_omni.platforms import current_omni_platform
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
if TYPE_CHECKING:
from .cached_code_predictor import FrameLocalKVCache
logger = init_logger(__name__)
# Backward-compat alias used by tests
Qwen3TTSTalkerCodePredictorModelVLLM = CodePredictorBaseModel
@@ -48,6 +58,82 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(CodePredictorWrapp
# Store talker_config for backward compat (accessed by some callers)
self.talker_config = talker_config
self._vllm_config = vllm_config
extra = self._stage_connector_extra_config(vllm_config)
self._kv_requested = self._parse_bool_config(extra.get("code_predictor_kv_cache"))
self._fused_sampling = self._parse_bool_config(extra.get("code_predictor_fused_sampling"))
self._frame_cache: FrameLocalKVCache | None = None
def _setup_compile(self) -> None:
if self._compiled_model_fwd is not None:
return
weight = next(self.model.parameters())
supported = (
current_omni_platform.is_cuda()
and weight.device.type == "cuda"
and weight.dtype == torch.bfloat16
and current_omni_platform.supports_torch_inductor()
)
if not self._kv_requested or not supported:
if self._kv_requested:
logger.warning("Qwen3-TTS frame-local KV requires CUDA BF16; using re-prefill")
return super()._setup_compile()
# Import CUDA-only kernels only when this model explicitly opts in.
from .cached_code_predictor import FrameLocalKVCache
self._model_dtype = weight.dtype
self._lm_heads_list = list(self.lm_head)
self._codec_embeds_list = list(self.model.codec_embedding)
self._bucket_sizes = self._batch_bucket_sizes()
max_batch = max(self._bucket_sizes)
self._ensure_buffers(weight.device, weight.dtype, max_batch)
cache = FrameLocalKVCache(self.model, max_batch)
with torch._dynamo.config.patch(
cache_size_limit=max(torch._dynamo.config.cache_size_limit, 2 * len(self._bucket_sizes))
):
for batch in self._bucket_sizes:
for step in range(1, min(3, self._num_groups)):
for _ in range(2):
cache(self._proj_buf, batch, step)
self._synchronize_warmup(weight.device)
self._frame_cache = cache
# Preserve the shared wrapper's initialization sentinel. Its full
# re-prefill callable is unused while _frame_cache is active.
self._compiled_model_fwd = self.model.forward
logger.info("Qwen3-TTS frame-local KV warmed for batches %s", self._bucket_sizes)
def _predict_step_logits(
self,
proj_buf: torch.Tensor,
bsz: int,
padded_bsz: int,
step: int,
is_npu_capturing: bool,
) -> torch.Tensor:
if self._frame_cache is None:
return super()._predict_step_logits(proj_buf, bsz, padded_bsz, step, is_npu_capturing)
hidden = self._frame_cache(proj_buf, padded_bsz, step)
return self._lm_heads_list[step - 1](hidden[:bsz, -1])
def _sample_per_call(
self,
logits: torch.Tensor,
inv_temperature: float,
top_k: int,
generator: torch.Generator | Sequence[torch.Generator | None] | None,
uniforms: torch.Tensor | None,
) -> torch.Tensor:
if (
self._fused_sampling
and current_omni_platform.is_cuda()
and logits.is_cuda
and uniforms is not None
and uniforms.dtype == torch.float32
):
from .code_predictor_sampling import sample_code_topk_gumbel
return sample_code_topk_gumbel(logits, uniforms, top_k, inv_temperature)
return super()._sample_per_call(logits, inv_temperature, top_k, generator, uniforms)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with vllm config context (required for VocabParallelEmbedding)."""
@@ -36,6 +36,7 @@ from vllm_omni.utils.speaker_cache import (
from vllm_omni.worker.sampling_utils import get_tts_local_seed
from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig
from .first_audio import talker_first_audio_enabled
from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder, resolve_x_vector_only
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
@@ -361,6 +362,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
}
)
# Set by the MRV2 model state once eager frames are enabled.
eager_frames_active = False
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
@@ -404,6 +408,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
# per-request additional_information.
self.talker_mtp_output_key = ("codes", "audio")
self.talker_mtp_graph_safe = True
# A frame's residual codebooks depend only on the Talker hidden and
# CB0 of the step that sampled it, so MRV2 may complete the frame at
# the end of that step instead of in the next step's preprocess.
self.mtp_eager_frames = talker_first_audio_enabled(vllm_config)
# The runners bypass only the outer whole-MTP graph when explicit
# generators are present, so seeded requests can still share one raw
# batched MTP call with independent per-row streams.
@@ -722,6 +730,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
logger.warning_once("runtime_additional_information is deprecated, use model_intermediate_buffer")
model_config = getattr(getattr(self, "vllm_config", None), "model_config", None)
async_chunk = bool(getattr(model_config, "async_chunk", False))
spans = kwargs.get("request_token_spans")
if async_chunk and self.eager_frames_active and spans is not None:
return self._make_eager_omni_output(hidden, info_dicts, spans)
audio_codes_list: list[torch.Tensor] = []
ref_code_len_segments: list[tuple[int | torch.Tensor, int, torch.device]] = []
ref_code_list: list[torch.Tensor] = []
@@ -863,6 +874,44 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
mm.setdefault("meta", {})["codec_frame_valid"] = codec_frame_valid[:span_len]
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
def _make_eager_omni_output(
self,
hidden: torch.Tensor,
info_dicts: list[Any],
spans: list[tuple[int, int]],
) -> OmniOutput:
"""Async-chunk output when the runner fills frames after sampling.
Every row that sampled this step gets its codes and validity from the
runner's eager MTP; the rest (non-final prefill chunks) carry no frame,
so the per-request buffer rows need not be gathered here.
"""
num_tokens = spans[-1][1] if spans else 0
q = int(self.talker_config.num_code_groups)
mm: OmniPayload = {
"codes": {"audio": torch.zeros((num_tokens, q), dtype=torch.long, device=hidden.device)},
"meta": {
"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8, device=hidden.device),
"first_audio": torch.zeros((num_tokens,), dtype=torch.int8, device=hidden.device),
},
}
ref_rows = [
index
for index, info in enumerate(info_dicts)
if isinstance(info, dict)
and isinstance(ref := info.get("codes", {}).get("ref"), torch.Tensor)
and ref.numel() > 0
and _should_publish_async_ref_codes(info)
]
if ref_rows:
empty = torch.empty(0, dtype=torch.long)
ref_code_list = [empty] * len(info_dicts)
for index in ref_rows:
ref_code_list[index] = info_dicts[index]["codes"]["ref"]
_mark_async_ref_codes_published(info_dicts[index])
mm["codes"]["ref"] = ref_code_list
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
# -------------------- preprocess / postprocess --------------------
def preprocess(
@@ -1087,7 +1136,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
f"preprocess_decode_batch expected {len(req_infos)} input ids, got {int(input_ids_flat.numel())}"
)
codec_frame_valid_batch = (input_ids_flat >= 0) & (input_ids_flat < self._codebook_vocab_size)
eager = self.eager_frames_active
codec_frame_valid_batch = (
None if eager else (input_ids_flat >= 0) & (input_ids_flat < self._codebook_vocab_size)
)
device = input_ids_flat.device
dtype = self._embedding_dtype
# Request-independent constant (see :meth:`_init_runtime_buffers`) —
@@ -1150,28 +1202,29 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
text_step = tts_pad_embed
next_text_offset = text_offset
last_hidden = hs.get("last")
if isinstance(last_hidden, torch.Tensor):
past_hidden = last_hidden.to(device=device, dtype=dtype).reshape(1, -1)
else:
# Match scalar preprocess(): EOS / async-scheduling races can
# arrive before postprocess has written last; zeros are filtered
# downstream the same way as the single-request path.
logger.warning_once(
"Qwen3-TTS batched decode missing last hidden; zero-filling past_hidden "
"to match scalar preprocess()"
)
past_hidden = torch.zeros_like(text_step)
past_hidden_list.append(past_hidden)
text_step_list.append(text_step)
info_update: dict[str, Any] = {
"meta": {
"talker_text_offset": int(next_text_offset),
"codec_streaming": codec_streaming,
"codec_frame_valid": codec_frame_valid_batch[len(updates)].reshape(()),
},
}
if not eager:
# Eager frames finished MTP last step: no hidden input or row validity needed.
last_hidden = hs.get("last")
if isinstance(last_hidden, torch.Tensor):
past_hidden = last_hidden.to(device=device, dtype=dtype).reshape(1, -1)
else:
# Match scalar preprocess(): EOS / async-scheduling races can
# arrive before postprocess has written last; zeros are filtered
# downstream the same way as the single-request path.
logger.warning_once(
"Qwen3-TTS batched decode missing last hidden; zero-filling past_hidden "
"to match scalar preprocess()"
)
past_hidden = torch.zeros_like(text_step)
past_hidden_list.append(past_hidden)
info_update["meta"]["codec_frame_valid"] = codec_frame_valid_batch[len(updates)].reshape(())
if trailing_text_update is not None:
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
updates.append(info_update)
@@ -1179,11 +1232,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
if input_embeds is None:
input_embeds = self.embed_input_ids(input_ids_flat.reshape(-1, 1).to(torch.long))
inputs_embeds_out = input_embeds.to(device=device, dtype=dtype).reshape(len(req_infos), -1)
text_steps = torch.cat(text_step_list, dim=0)
return (
input_ids_flat,
inputs_embeds_out,
torch.cat(past_hidden_list, dim=0),
torch.cat(text_step_list, dim=0),
text_steps if eager else torch.cat(past_hidden_list, dim=0),
text_steps,
updates,
)
@@ -1382,6 +1436,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
logger.info("Loaded %d weights for Qwen3TTSTalkerForConditionalGeneration", len(loaded))
self._build_stacked_codec_embed()
if talker_first_audio_enabled(self.vllm_config):
from .first_frame_decoder import Qwen3TTSFirstFrameDecoder
self.first_frame_decoder = Qwen3TTSFirstFrameDecoder(self.model_path)
decoder_loaded = self.first_frame_decoder.load(self.vllm_config)
loaded = set(loaded) | {f"first_frame_decoder.{name}" for name in decoder_loaded}
return loaded
def _build_stacked_codec_embed(self) -> None:
@@ -1569,6 +1629,35 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
mtp = talker_mtp
get_mtp_seed = staticmethod(get_tts_local_seed)
# Postprocess only stores the last hidden for the deferred decode MTP.
eager_frames_need_postprocess = False
def eager_decode_settled(self, info: dict[str, Any]) -> bool:
"""Decode preprocess is a no-op for this row: its trailing text is spent.
From then on every decode step feeds the frame embedding plus the pad
text step and changes no per-request state (the non-streaming
CustomVoice/VoiceDesign case after the first decode).
"""
hs = info.get("hidden_states")
tail = hs.get("trailing_text") if isinstance(hs, dict) else None
meta = info.get("meta")
offset = meta.get("talker_text_offset", 0) if isinstance(meta, dict) else 0
return isinstance(tail, torch.Tensor) and tail.numel() == 0 and not offset
def eager_settled_text_step(self) -> torch.Tensor:
return self._tts_pad_embed.reshape(1, -1)
def capture_first_frame_graphs(self) -> None:
# Set by load_weights when talker_first_audio_enabled accepts the connector option.
decoder = getattr(self, "first_frame_decoder", None)
if decoder is not None:
decoder.capture()
def mtp_frame_valid(self, layer0: torch.Tensor) -> torch.Tensor:
"""Rows whose CB0 is a codec id rather than EOS/special; matches decode preprocess."""
return (layer0 >= 0) & (layer0 < self._codebook_vocab_size)
@property
def mtp_output_key(self) -> tuple[str, str]:
return self.talker_mtp_output_key
@@ -71,6 +71,7 @@ class CUDAGraphDecoderWrapper:
self.combined_states: dict[tuple[str, int, int], dict] = {}
self.icl_prefix_states: dict[int, dict] = {}
self.xvec_prefix_states: dict[int, dict] = {}
self.xvec_prefix_state_only_states: dict[int, dict] = {}
self.stateless_states: dict[tuple[int, int], dict] = {}
self._device = None
@@ -230,6 +231,7 @@ class CUDAGraphDecoderWrapper:
*self._retained_kv_caches(self.combined_states),
*self._retained_kv_caches(self.icl_prefix_states),
*self._retained_kv_caches(self.xvec_prefix_states),
*self._retained_kv_caches(self.xvec_prefix_state_only_states),
]
batched_captures = sum(rows for rows, _ in retained)
@@ -248,7 +250,7 @@ class CUDAGraphDecoderWrapper:
rows_reason=(
f"rows across {len(retained)} retained caches in "
f"{len(self.combined_states)} suffix / {len(self.icl_prefix_states)} icl-prefix / "
f"{len(self.xvec_prefix_states)} xvec-prefix captures"
f"{len(self.xvec_prefix_states) + len(self.xvec_prefix_state_only_states)} xvec-prefix captures"
),
allocation_note=(
"counted from caches transformers has materialized; captures whose tensors are still "
@@ -424,6 +426,14 @@ class CUDAGraphDecoderWrapper:
logger.info("Captured xvec prefix CUDA Graph for batch=%d", batch_size)
except Exception:
logger.warning("Failed to capture xvec prefix CUDA Graph for batch=%d", batch_size, exc_info=True)
if getattr(self.decoder, "capture_first_audio_state_only", False) and self.initial_chunk_frames == 1:
try:
self._capture_xvec_prefix(batch_size, device, dtype, state_only=True)
logger.info("Captured xvec state-only CUDA Graph for batch=%d", batch_size)
except Exception:
logger.warning(
"Failed to capture xvec state-only graph for batch=%d", batch_size, exc_info=True
)
logger.info(
"Starting CUDA Graph warmup for %d shapes: batch_sizes=%s seq_lens=%s",
@@ -631,7 +641,9 @@ class CUDAGraphDecoderWrapper:
"cache": caches,
}
def _capture_xvec_prefix(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> None:
def _capture_xvec_prefix(
self, batch_size: int, device: torch.device, dtype: torch.dtype, *, state_only: bool = False
) -> None:
static_input = torch.zeros(
batch_size,
self.num_quantizers,
@@ -639,8 +651,13 @@ class CUDAGraphDecoderWrapper:
dtype=dtype,
device=device,
)
# The Talker delivers exactly one frame upstream: a one-frame first
# chunk then only needs its state; a longer one keeps the other frames.
first_chunk = (
self.decoder._decode_xvec_first_chunk_state_only if state_only else self.decoder._decode_xvec_first_chunk
)
with torch.no_grad():
_ = self.decoder._decode_xvec_first_chunk(static_input, {})
_ = first_chunk(static_input, {})
torch.accelerator.synchronize(device)
config = self.decoder.config
@@ -662,13 +679,15 @@ class CUDAGraphDecoderWrapper:
graph = CUDAGraph()
with torch.no_grad():
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_output = self.decoder._decode_xvec_first_chunk(static_input, caches)
static_output = first_chunk(static_input, caches)
self.xvec_prefix_states[batch_size] = {
states = self.xvec_prefix_state_only_states if state_only else self.xvec_prefix_states
states[batch_size] = {
"graph": graph,
"input": {"codes": static_input},
"output": static_output,
"cache": caches,
"state_only": state_only,
}
def _ensure_suffix_buffers(self, caches: dict) -> None:
@@ -808,7 +827,14 @@ class CUDAGraphDecoderWrapper:
codes_list: list[torch.Tensor],
request_caches: list[dict],
) -> list[torch.Tensor] | None:
available = set(getattr(self, "xvec_prefix_states", {}))
skip_flags = [bool(cache.get("skip_first_audio", False)) for cache in request_caches]
# Full PCM graphs remain available even when the Talker cannot use
# direct delivery (or only some requests qualify). The option is only
# a capture hint; request metadata decides which audio is still owed.
states = getattr(self, "xvec_prefix_states", {})
if skip_flags and all(skip_flags):
states = getattr(self, "xvec_prefix_state_only_states", {}) or states
available = set(states)
if not available:
self._record_graph_fallback("xvec_prefix:no_graph", len(codes_list))
return None
@@ -835,7 +861,7 @@ class CUDAGraphDecoderWrapper:
self._record_graph_fallback("xvec_prefix:no_graph", len(codes_list))
return None
state = self.xvec_prefix_states[batch_size]
state = states[batch_size]
static_input = state["input"]["codes"]
static_input.zero_()
for row, codes in enumerate(codes_list):
@@ -855,7 +881,12 @@ class CUDAGraphDecoderWrapper:
cache["suffix_conv"] = static_caches["suffix_conv"][row : row + 1].clone()
cache["suffix_frames"] = self.initial_chunk_frames
self._ensure_suffix_buffers(cache)
outputs.append(state["output"][row : row + 1].clone())
if state.get("state_only"):
outputs.append(static_input.new_zeros((1, 1, 0), dtype=torch.float32))
elif skip_flags[row]:
outputs.append(state["output"][row : row + 1, :, self.decoder.total_upsample :].clone())
else:
outputs.append(state["output"][row : row + 1].clone())
return outputs
def _decode_suffix_batch(
@@ -983,6 +1014,8 @@ class CUDAGraphDecoderWrapper:
def _decode_request_fallback(self, codes: torch.Tensor, cache: dict) -> torch.Tensor:
if "suffix_quantized" not in cache:
if int(cache["prefix_frames"]) == 0:
if cache.get("skip_first_audio", False):
return self.decoder._decode_stream_first_chunk(codes, cache)
return self.decoder._decode_xvec_first_chunk(codes, cache)
prefix_frames = int(cache["prefix_frames"])
output = self.decoder._decode_icl_first_chunk(codes, cache, prefix_frames)
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Short causal attention for the scratch KV cache inside a code predictor."""
import torch
from vllm.triton_utils import tl, triton
@triton.jit
def _short_kv_attention(
q_ptr,
k_ptr,
v_ptr,
positions_ptr,
out_ptr,
q_stride_b: tl.constexpr,
q_stride_h: tl.constexpr,
q_stride_n: tl.constexpr,
k_stride_b: tl.constexpr,
k_stride_h: tl.constexpr,
k_stride_n: tl.constexpr,
v_stride_b: tl.constexpr,
v_stride_h: tl.constexpr,
v_stride_n: tl.constexpr,
heads: tl.constexpr,
num_queries: tl.constexpr,
num_keys: tl.constexpr,
head_dim: tl.constexpr,
queries_per_kv: tl.constexpr,
scale: tl.constexpr,
block_n: tl.constexpr,
block_d: tl.constexpr,
):
b = tl.program_id(0)
h = tl.program_id(1)
qi = tl.program_id(2)
kvh = h // queries_per_kv
d = tl.arange(0, block_d)
n = tl.arange(0, block_n)
last = tl.load(positions_ptr + qi)
q = tl.load(q_ptr + b * q_stride_b + h * q_stride_h + qi * q_stride_n + d, d < head_dim, other=0).to(tl.float32)
k = tl.load(
k_ptr + b * k_stride_b + kvh * k_stride_h + n[:, None] * k_stride_n + d[None, :],
(n[:, None] < num_keys) & (n[:, None] <= last) & (d[None, :] < head_dim),
other=0,
).to(tl.float32)
score = tl.sum(k * q[None, :], 1) * scale
score = tl.where((n < num_keys) & (n <= last), score, -float("inf"))
p = tl.exp(score - tl.max(score, 0))
p = p / tl.sum(p, 0)
v = tl.load(
v_ptr + b * v_stride_b + kvh * v_stride_h + n[:, None] * v_stride_n + d[None, :],
(n[:, None] < num_keys) & (n[:, None] <= last) & (d[None, :] < head_dim),
other=0,
).to(tl.float32)
out = tl.sum(p[:, None] * v, 0)
tl.store(out_ptr + ((b * num_queries + qi) * heads + h) * head_dim + d, out, d < head_dim)
def short_kv_attention(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, positions: torch.Tensor, scale: float
) -> torch.Tensor:
"""One/two-query GQA over a frame-local scratch cache with contiguous head dimensions.
The returned [B, H, Q, D] tensor has a contiguous [B, Q, H, D] view,
avoiding a separate transpose-copy before the output projection.
"""
b, h, nq, d = q.shape
nk = k.shape[2]
out = torch.empty((b, nq, h, d), device=q.device, dtype=q.dtype)
_short_kv_attention[(b, h, nq)](
q,
k,
v,
positions,
out,
*q.stride()[:3],
*k.stride()[:3],
*v.stride()[:3],
h,
nq,
nk,
d,
h // k.shape[1],
scale,
triton.next_power_of_2(nk),
triton.next_power_of_2(d),
num_warps=4,
)
return out.transpose(1, 2)
@@ -997,6 +997,33 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
if count > 0:
logger.info("Precomputed exp caches for %d SnakeBeta activations", count)
# Set by enable_time_major_conv(); a plain object so it stays out of state_dict().
_time_major_stack = None
def enable_time_major_conv(self) -> None:
"""Run the upsample + decoder convs time-major: one tensor-core GEMM per conv.
Call after the decoder reaches its device and dtype (the stack keeps
GEMM-layout weight copies) and before CUDA graph capture.
"""
from .time_major_stack import TimeMajorConvStack
self._time_major_stack = TimeMajorConvStack(self)
logger.info("Qwen3-TTS decoder: time-major (GEMM) conv stack enabled")
def _conv_decode(self, hidden: torch.Tensor) -> torch.Tensor:
"""Pre-transformer output ``[B, T, latent]`` -> unclamped waveform ``[B, 1, T * total_upsample]``."""
if self._time_major_stack is not None:
return self._time_major_stack(hidden)
hidden = hidden.permute(0, 2, 1)
for blocks in self.upsample:
for block in blocks:
hidden = block(hidden)
wav = hidden
for block in self.decoder:
wav = block(wav)
return wav
def enable_cudagraph(
self,
capture_modes: tuple[str, ...] = ("icl", "xvec"),
@@ -1056,15 +1083,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
hidden = self.pre_conv(hidden).transpose(1, 2)
hidden = self.pre_transformer(inputs_embeds=hidden).last_hidden_state
hidden = hidden.permute(0, 2, 1)
for blocks in self.upsample:
for block in blocks:
hidden = block(hidden)
wav = hidden
for block in self.decoder:
wav = block(wav)
return wav.clamp(min=-1, max=1)
return self._conv_decode(hidden).clamp(min=-1, max=1)
def _decode_icl_first_chunk(
self,
@@ -1126,20 +1145,32 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
caches.update({"past_key_values": prefix_cache})
caches.update({"prefix_hidden": prefix_hidden})
hidden = hidden.permute(0, 2, 1)
for blocks in self.upsample:
for block in blocks:
hidden = block(hidden)
wav = hidden
for block in self.decoder:
wav = block(wav)
wav = wav.clamp(min=-1, max=1)
return wav
return self._conv_decode(hidden).clamp(min=-1, max=1)
def _decode_xvec_first_chunk(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
"""Initialize prefixless incremental state from the first codec chunk."""
suffix_hidden = self._init_xvec_first_chunk_state(codes, caches)
return self._conv_decode(suffix_hidden).clamp(min=-1, max=1)
def _decode_xvec_first_chunk_state_only(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
"""State for later chunks when the first chunk's audio was delivered upstream."""
return self._init_xvec_first_chunk_state(codes, caches)
# Capture hint only: the raw connector option does not guarantee that
# a request received upstream audio. Only its skip_first_audio marker does.
capture_first_audio_state_only = False
def _decode_stream_first_chunk(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
if not caches.get("skip_first_audio", False):
return self._decode_xvec_first_chunk(codes, caches)
if int(codes.shape[-1]) <= 1:
self._decode_xvec_first_chunk_state_only(codes, caches)
return codes.new_zeros((codes.shape[0], 1, 0), dtype=torch.float32)
# The Talker delivered only the first frame; a longer first chunk
# (adaptive or ramped schedules) still owes the audio of the rest.
return self._decode_xvec_first_chunk(codes, caches)[..., self.total_upsample :]
def _init_xvec_first_chunk_state(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
hidden = self.quantizer.decode(codes)
caches["decoder_prefix_frames"] = 0
caches["ref_hidden"] = hidden[:, :, :0]
@@ -1172,16 +1203,8 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
).last_hidden_state
caches["past_key_values"] = empty_prefix_cache
caches["prefix_hidden"] = suffix_hidden[:, :0, :]
hidden = suffix_hidden.permute(0, 2, 1)
for blocks in self.upsample:
for block in blocks:
hidden = block(hidden)
wav = hidden
for block in self.decoder:
wav = block(wav)
caches["suffix_frames"] = int(codes.shape[-1])
return wav.clamp(min=-1, max=1)
return suffix_hidden
def _decode_suffix(
self,
@@ -1229,14 +1252,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
hidden = torch.cat([caches["prefix_hidden"], suffix_hidden], dim=1)
required_frames = min(hidden.shape[1], new_frames + _DOWNSTREAM_CONTEXT_FRAME)
hidden = hidden[:, -required_frames:, :].permute(0, 2, 1)
for blocks in self.upsample:
for block in blocks:
hidden = block(hidden)
wav = hidden
for block in self.decoder:
wav = block(wav)
wav = self._conv_decode(hidden[:, -required_frames:, :])
wav = wav[..., -new_frames * self.total_upsample :].clamp(min=-1, max=1)
return wav, next_quantized, next_conv
@@ -1300,7 +1316,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
f"prefix_frames={prefix_frames}, total_frames={codes.shape[-1]}"
)
if prefix_frames == 0:
return self._decode_xvec_first_chunk(codes, caches)
return self._decode_stream_first_chunk(codes, caches)
return self._decode_icl_first_chunk(codes, caches, prefix_frames)
else:
decoder_prefix_frames = int(caches.get("decoder_prefix_frames", prefix_frames))
@@ -1546,7 +1562,11 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
return None
batched_codes = torch.cat(codes_list, dim=0)
batched_cache: dict[str, Any] = {}
output = self._decode_xvec_first_chunk(batched_codes, batched_cache)
skip_flags = {bool(cache.get("skip_first_audio", False)) for cache in request_caches}
if len(skip_flags) != 1:
return None
batched_cache["skip_first_audio"] = skip_flags.pop()
output = self._decode_stream_first_chunk(batched_codes, batched_cache)
outputs: list[torch.Tensor] = []
for row, cache in enumerate(request_caches):
for key in ("ref_hidden", "ref_conv", "prefix_hidden", "suffix_quantized", "suffix_conv"):
@@ -0,0 +1,347 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Time-major causal 1-D convolution for streaming vocoders.
Activations are ``[B, T, C]`` so that every dense convolution is one GEMM over
``B * T`` rows (bf16 tensor cores through cuBLAS) instead of an NCT cuDNN
convolution, which for these long, thin decoder layers picks non-tensor-core
kernels and adds layout transforms around each call.
- :func:`snake_im2col` gathers each output step's causal, dilated window into a
``[B * T, K * C]`` row (tap-major), optionally applying ``bias`` and SnakeBeta
on the way in; ``K = 1`` is a standalone (bias +) SnakeBeta.
- :func:`causal_conv` is the dense causal (dilated) convolution as an implicit
GEMM: each K-block of the reduction reads the input rows shifted by one tap,
so the ``K * C`` im2col matrix is never materialized; bias and an optional
residual are added in the epilogue.
- :func:`transposed_overlap_add` finishes a transposed convolution of kernel
``stride`` or ``2 * stride`` whose GEMM produced ``[B, T, K, C_out]``: the
overlap-add of consecutive inputs, the bias and the causal right trim.
Weights come from ``nn.Conv1d``/``nn.ConvTranspose1d`` via :func:`conv_gemm_weight`
and :func:`transposed_gemm_weight`.
"""
from __future__ import annotations
import torch
from vllm.triton_utils import HAS_TRITON, tl, triton
def conv_gemm_weight(weight: torch.Tensor) -> torch.Tensor:
"""``nn.Conv1d`` weight ``[C_out, C_in, K]`` -> tap-major GEMM weight ``[K * C_in, C_out]``."""
c_out, c_in, k = weight.shape
return weight.detach().permute(2, 1, 0).reshape(k * c_in, c_out).contiguous()
def transposed_gemm_weight(weight: torch.Tensor) -> torch.Tensor:
"""``nn.ConvTranspose1d`` weight ``[C_in, C_out, K]`` -> GEMM weight ``[C_in, K * C_out]``."""
c_in, c_out, k = weight.shape
return weight.detach().permute(0, 2, 1).reshape(c_in, k * c_out).contiguous()
def snake_im2col_reference(
x: torch.Tensor,
kernel_size: int,
dilation: int = 1,
bias: torch.Tensor | None = None,
snake: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
"""Eager equivalent of :func:`snake_im2col` (also the CPU path)."""
if bias is not None:
x = x + bias
if snake is not None:
exp_alpha, inv_beta = snake
x = x + (inv_beta.float() * torch.sin(x.float() * exp_alpha.float()).square()).to(x.dtype)
if kernel_size == 1:
return x.reshape(-1, x.shape[-1])
t = x.shape[1]
xp = torch.nn.functional.pad(x, (0, 0, (kernel_size - 1) * dilation, 0))
cols = torch.cat([xp[:, j * dilation : j * dilation + t] for j in range(kernel_size)], dim=-1)
return cols.reshape(-1, cols.shape[-1])
def causal_conv_reference(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
kernel_size: int,
dilation: int = 1,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
"""Eager equivalent of :func:`causal_conv`: ``[B, T, C]`` -> ``[B, T, C_out]``."""
bsz, t, _ = x.shape
cols = snake_im2col_reference(x, kernel_size, dilation)
out = cols @ weight if bias is None else torch.addmm(bias, cols, weight)
out = out.view(bsz, t, -1)
return out if residual is None else out + residual
def transposed_overlap_add_reference(z: torch.Tensor, stride: int, bias: torch.Tensor) -> torch.Tensor:
"""Eager equivalent of :func:`transposed_overlap_add`."""
bsz, t, k, c_out = z.shape
out = z[:, :, :stride]
if k > stride:
out = out + torch.nn.functional.pad(z[:, :-1, stride:], (0, 0, 0, 0, 1, 0))
return out.reshape(bsz, t * stride, c_out) + bias
if HAS_TRITON:
@triton.jit
def _snake_im2col_kernel(
x_ptr,
bias_ptr,
exp_alpha_ptr,
inv_beta_ptr,
out_ptr,
n_rows,
t_len,
c_len,
dilation,
K: tl.constexpr, # noqa: N803
HAS_BIAS: tl.constexpr, # noqa: N803
HAS_SNAKE: tl.constexpr, # noqa: N803
BLOCK_R: tl.constexpr, # noqa: N803
BLOCK_C: tl.constexpr, # noqa: N803
):
rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
chans = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
row_mask = rows < n_rows
chan_mask = chans < c_len
t = rows % t_len
if HAS_BIAS:
bias = tl.load(bias_ptr + chans, mask=chan_mask, other=0.0)
if HAS_SNAKE:
exp_alpha = tl.load(exp_alpha_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)
inv_beta = tl.load(inv_beta_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)
out_mask = row_mask[:, None] & chan_mask[None, :]
for j in tl.static_range(K):
shift = (K - 1 - j) * dilation
valid = row_mask & (t >= shift)
src = rows - shift
x = tl.load(
x_ptr + src[:, None].to(tl.int64) * c_len + chans[None, :],
mask=valid[:, None] & chan_mask[None, :],
other=0.0,
)
if HAS_BIAS:
# Causal padding is applied to the conv input, after bias.
x = tl.where(valid[:, None], x + bias[None, :], 0.0).to(x_ptr.dtype.element_ty)
if HAS_SNAKE:
s = tl.sin(x.to(tl.float32) * exp_alpha[None, :])
x = x + (inv_beta[None, :] * s * s).to(x.dtype)
tl.store(out_ptr + rows[:, None].to(tl.int64) * (K * c_len) + j * c_len + chans[None, :], x, mask=out_mask)
@triton.autotune(
configs=[
triton.Config({"BLOCK_M": bm, "BLOCK_N": bn, "BLOCK_K": bk}, num_warps=w, num_stages=st)
for bm, bn, bk, w, st in (
(128, 128, 64, 8, 3),
(128, 64, 64, 4, 4),
(64, 128, 64, 4, 4),
(128, 32, 32, 4, 4),
(64, 64, 32, 4, 4),
(256, 32, 32, 8, 3),
(256, 64, 32, 8, 3),
(128, 16, 32, 4, 4),
)
],
key=["row_bucket", "n_cols", "c_len", "K", "dilation"],
)
@triton.jit
def _causal_conv_gemm_kernel(
x_ptr,
w_ptr,
bias_ptr,
res_ptr,
out_ptr,
n_rows,
n_cols,
c_len,
t_len,
dilation,
row_bucket, # autotune key only: power-of-two bucket of n_rows
K: tl.constexpr, # noqa: N803
HAS_BIAS: tl.constexpr, # noqa: N803
HAS_RES: tl.constexpr, # noqa: N803
PRECISION: tl.constexpr, # noqa: N803
BLOCK_M: tl.constexpr, # noqa: N803
BLOCK_N: tl.constexpr, # noqa: N803
BLOCK_K: tl.constexpr, # noqa: N803
):
# Row-major tiles, N fastest: neighbouring programs share the input rows.
pid = tl.program_id(0)
num_n = tl.cdiv(n_cols, BLOCK_N)
pid_m = pid // num_n
pid_n = pid % num_n
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
row_mask = rows < n_rows
col_mask = cols < n_cols
t = rows % t_len
k_blocks = tl.cdiv(c_len, BLOCK_K)
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for kk in range(0, K * k_blocks):
tap = kk // k_blocks
chans = (kk % k_blocks) * BLOCK_K + tl.arange(0, BLOCK_K)
chan_mask = chans < c_len
shift = (K - 1 - tap) * dilation
a = tl.load(
x_ptr + (rows - shift)[:, None] * c_len + chans[None, :],
mask=(row_mask & (t >= shift))[:, None] & chan_mask[None, :],
other=0.0,
)
b = tl.load(
w_ptr + (tap * c_len + chans)[:, None] * n_cols + cols[None, :],
mask=chan_mask[:, None] & col_mask[None, :],
other=0.0,
)
acc = tl.dot(a, b, acc, input_precision=PRECISION)
if HAS_BIAS:
acc += tl.load(bias_ptr + cols, mask=col_mask, other=0.0).to(tl.float32)[None, :]
out_offsets = rows[:, None] * n_cols + cols[None, :]
out_mask = row_mask[:, None] & col_mask[None, :]
if HAS_RES:
acc += tl.load(res_ptr + out_offsets, mask=out_mask, other=0.0).to(tl.float32)
tl.store(out_ptr + out_offsets, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
@triton.jit
def _transposed_overlap_add_kernel(
z_ptr,
bias_ptr,
out_ptr,
n_out_rows,
t_len,
c_len,
STRIDE: tl.constexpr, # noqa: N803
K: tl.constexpr, # noqa: N803
BLOCK_R: tl.constexpr, # noqa: N803
BLOCK_C: tl.constexpr, # noqa: N803
):
rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R) # over B * T * STRIDE output steps
chans = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
row_mask = rows < n_out_rows
chan_mask = chans < c_len
mask = row_mask[:, None] & chan_mask[None, :]
tap = rows % STRIDE
src = rows // STRIDE # input row b * T + t
acc = tl.load(
z_ptr + (src[:, None].to(tl.int64) * K + tap[:, None]) * c_len + chans[None, :], mask=mask, other=0.0
).to(tl.float32)
if K > STRIDE:
prev_valid = row_mask & (src % t_len > 0)
acc += tl.load(
z_ptr + ((src[:, None].to(tl.int64) - 1) * K + tap[:, None] + STRIDE) * c_len + chans[None, :],
mask=prev_valid[:, None] & chan_mask[None, :],
other=0.0,
).to(tl.float32)
acc += tl.load(bias_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)[None, :]
tl.store(
out_ptr + rows[:, None].to(tl.int64) * c_len + chans[None, :], acc.to(out_ptr.dtype.element_ty), mask=mask
)
def _blocks(c_len: int) -> tuple[int, int]:
block_c = min(128, triton.next_power_of_2(c_len))
return max(1, 4096 // block_c), block_c
def snake_im2col(
x: torch.Tensor,
kernel_size: int,
dilation: int = 1,
bias: torch.Tensor | None = None,
snake: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
"""``[B, T, C]`` -> ``[B * T, K * C]`` causal im2col with optional bias + SnakeBeta."""
if not HAS_TRITON or not x.is_cuda:
return snake_im2col_reference(x, kernel_size, dilation, bias, snake)
x = x.contiguous()
bsz, t, c = x.shape
n_rows = bsz * t
out = torch.empty(n_rows, kernel_size * c, dtype=x.dtype, device=x.device)
block_r, block_c = _blocks(c)
exp_alpha, inv_beta = snake if snake is not None else (x, x)
_snake_im2col_kernel[(triton.cdiv(n_rows, block_r), triton.cdiv(c, block_c))](
x,
bias if bias is not None else x,
exp_alpha,
inv_beta,
out,
n_rows,
t,
c,
dilation,
K=kernel_size,
HAS_BIAS=bias is not None,
HAS_SNAKE=snake is not None,
BLOCK_R=block_r,
BLOCK_C=block_c,
)
return out
def causal_conv(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
kernel_size: int,
dilation: int = 1,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
"""Causal dilated conv ``[B, T, C]`` -> ``[B, T, C_out]`` with a :func:`conv_gemm_weight` weight.
``residual`` (``[B, T, C_out]``) is added in the epilogue.
"""
if not HAS_TRITON or not x.is_cuda:
return causal_conv_reference(x, weight, bias, kernel_size, dilation, residual)
x = x.contiguous()
bsz, t, c = x.shape
n_rows, n_cols = bsz * t, weight.shape[1]
if weight.shape[0] != kernel_size * c:
raise ValueError(f"conv weight has {weight.shape[0]} rows, expected {kernel_size} taps x {c} channels")
if max(n_rows * max(c, n_cols), weight.numel()) >= 2**31:
raise ValueError("causal_conv offsets are 32-bit; split the batch")
out = torch.empty(bsz, t, n_cols, dtype=x.dtype, device=x.device)
if residual is not None:
residual = residual.contiguous()
def grid(meta):
return (triton.cdiv(n_rows, meta["BLOCK_M"]) * triton.cdiv(n_cols, meta["BLOCK_N"]),)
_causal_conv_gemm_kernel[grid](
x,
weight,
bias if bias is not None else x,
residual if residual is not None else x,
out,
n_rows,
n_cols,
c,
t,
dilation,
triton.next_power_of_2(triton.cdiv(n_rows, 1024)),
K=kernel_size,
HAS_BIAS=bias is not None,
HAS_RES=residual is not None,
PRECISION="ieee" if x.dtype == torch.float32 else "tf32",
)
return out
def transposed_overlap_add(z: torch.Tensor, stride: int, bias: torch.Tensor) -> torch.Tensor:
"""``[B, T, K, C_out]`` GEMM output of a kernel-``K`` transposed conv -> ``[B, T * stride, C_out]``."""
bsz, t, k, c_out = z.shape
if k not in (stride, 2 * stride):
raise ValueError(f"transposed_overlap_add supports kernel stride or 2*stride, got kernel={k} stride={stride}")
if not HAS_TRITON or not z.is_cuda:
return transposed_overlap_add_reference(z, stride, bias)
z = z.contiguous()
n_out_rows = bsz * t * stride
out = torch.empty(bsz, t * stride, c_out, dtype=z.dtype, device=z.device)
block_r, block_c = _blocks(c_out)
_transposed_overlap_add_kernel[(triton.cdiv(n_out_rows, block_r), triton.cdiv(c_out, block_c))](
z, bias, out, n_out_rows, t, c_out, STRIDE=stride, K=k, BLOCK_R=block_r, BLOCK_C=block_c
)
return out
@@ -0,0 +1,100 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Time-major upsample + decoder conv stack for the Qwen3-TTS 12 Hz codec.
Same math as ``Qwen3TTSTokenizerV2Decoder.upsample``/``.decoder`` on ``[B, T, C]``
activations: dense causal convs are implicit GEMMs (tensor cores, no im2col
buffer, residual adds in the epilogue), SnakeBeta one pass per activation,
transposed convs a GEMM plus an overlap-add. See ``time_major_conv.py``.
"""
from __future__ import annotations
import torch
from torch import nn
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.time_major_conv import (
causal_conv,
conv_gemm_weight,
snake_im2col,
transposed_gemm_weight,
transposed_overlap_add,
)
def _snake(module) -> tuple[torch.Tensor, torch.Tensor]:
if module._exp_alpha is None:
module.precompute_exp_cache()
return module._exp_alpha, module._inv_beta
class _Conv:
def __init__(self, causal_conv) -> None:
conv = causal_conv.conv
if conv.stride[0] != 1 or conv.groups != 1:
raise ValueError("time-major decoder convs must be dense with stride 1")
self.weight = conv_gemm_weight(conv.weight)
self.bias = conv.bias.detach()
self.kernel_size = conv.kernel_size[0]
self.dilation = conv.dilation[0]
def __call__(self, x: torch.Tensor, residual: torch.Tensor | None = None) -> torch.Tensor:
return causal_conv(x, self.weight, self.bias, self.kernel_size, self.dilation, residual)
class _Transposed:
def __init__(self, causal_trans_conv) -> None:
conv = causal_trans_conv.conv
self.weight = transposed_gemm_weight(conv.weight)
self.bias = conv.bias.detach()
self.kernel_size = conv.kernel_size[0]
self.stride = conv.stride[0]
self.out_channels = conv.out_channels
def __call__(self, x2d: torch.Tensor, bsz: int, t: int) -> torch.Tensor:
z = torch.mm(x2d, self.weight).view(bsz, t, self.kernel_size, self.out_channels)
return transposed_overlap_add(z, self.stride, self.bias)
class TimeMajorConvStack:
"""Callable replacing the NCT conv loops; holds GEMM-layout copies of the weights."""
def __init__(self, decoder: nn.Module) -> None:
self.upsample = [(_Transposed(trans), block) for trans, block in decoder.upsample]
self.conv_in = _Conv(decoder.decoder[0])
self.blocks = []
for block in decoder.decoder[1:-2]:
snake, trans, *units = block.block
residuals = [(_snake(u.act1), _Conv(u.conv1), _snake(u.act2), _Conv(u.conv2)) for u in units]
self.blocks.append((_snake(snake), _Transposed(trans), residuals))
self.snake_out = _snake(decoder.decoder[-2])
self.conv_out = _Conv(decoder.decoder[-1])
@staticmethod
def _convnext(x: torch.Tensor, block) -> torch.Tensor:
dw = block.dwconv.conv
k, t = dw.kernel_size[0], x.shape[1]
xp = torch.nn.functional.pad(x, (0, 0, k - 1, 0))
weight = dw.weight[:, 0, :].float()
h = dw.bias.float() + sum(xp[:, j : j + t].float() * weight[:, j] for j in range(k))
h = block.pwconv2(block.act(block.pwconv1(block.norm(h.to(x.dtype)))))
return x + block.gamma * h
def __call__(self, hidden: torch.Tensor) -> torch.Tensor:
"""``hidden`` ``[B, T, latent]`` -> unclamped waveform ``[B, 1, T * upsample]``."""
bsz = hidden.shape[0]
def snake(x, params):
return snake_im2col(x, 1, snake=params).view(x.shape)
for trans, block in self.upsample:
t = hidden.shape[1]
hidden = self._convnext(trans(hidden.reshape(bsz * t, -1), bsz, t), block)
hidden = self.conv_in(hidden)
for block_snake, trans, residuals in self.blocks:
t = hidden.shape[1]
hidden = trans(snake_im2col(hidden, 1, snake=block_snake), bsz, t)
for act1, conv1, act2, conv2 in residuals:
hidden = conv2(snake(conv1(snake(hidden, act1)), act2), residual=hidden)
wav = self.conv_out(snake(hidden, self.snake_out))
return wav.view(bsz, 1, -1)