[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
55 files changed
+4222
-230
No files matched your search
@@ -1077,6 +1077,64 @@ class CodePredictorWrapper(nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
def _predict_step_logits(
|
||||
self,
|
||||
proj_buf: torch.Tensor,
|
||||
bsz: int,
|
||||
padded_bsz: int,
|
||||
step: int,
|
||||
is_npu_capturing: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Run one residual-codebook step; model wrappers may specialize it."""
|
||||
device = proj_buf.device
|
||||
max_seq = self._num_groups + 1
|
||||
model_fwd = self._compiled_model_fwd
|
||||
lm_heads = self._lm_heads_list
|
||||
graph_key: int | tuple[int, int] = padded_bsz
|
||||
seq_len = max_seq
|
||||
if self._prefix_reprefill_enabled:
|
||||
actual_seq_len = step + 1
|
||||
for prefix_seq_len in self._prefix_reprefill_seq_lens:
|
||||
if prefix_seq_len < actual_seq_len:
|
||||
continue
|
||||
prefix_key = (padded_bsz, prefix_seq_len)
|
||||
if prefix_key in self._bucket_pos_ids:
|
||||
graph_key = prefix_key
|
||||
seq_len = prefix_seq_len
|
||||
break
|
||||
pos_ids = self._bucket_pos_ids.get(graph_key)
|
||||
if pos_ids is None:
|
||||
pos_ids = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
|
||||
)
|
||||
|
||||
# Use captured device graph if available, otherwise call compiled fn.
|
||||
device_graph_entry = self._device_graphs.get(graph_key)
|
||||
|
||||
# Let the outer graph record the regular forward during capture;
|
||||
# normal inference still uses the inner graph replay fast path.
|
||||
if device_graph_entry is not None and not is_npu_capturing:
|
||||
device_graph_entry[0].replay()
|
||||
hidden_out = device_graph_entry[1]
|
||||
else:
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
|
||||
|
||||
return lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
|
||||
def _sample_per_call(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
inv_temperature: float,
|
||||
top_k: int,
|
||||
generator: _GeneratorLike,
|
||||
uniforms: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
return self._sample_codes_gumbel(scaled, generator=generator, uniforms=uniforms)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
@@ -1114,10 +1172,7 @@ class CodePredictorWrapper(nn.Module):
|
||||
self._ensure_buffers(device, dtype, padded_bsz)
|
||||
|
||||
proj_buf = self._proj_buf
|
||||
max_seq = num_groups + 1
|
||||
projection = self.small_to_mtp_projection
|
||||
model_fwd = self._compiled_model_fwd
|
||||
lm_heads = self._lm_heads_list
|
||||
codec_embeds = self._codec_embeds_list
|
||||
# torch-npu cannot replay an inner NPUGraph while the outer talker_mtp
|
||||
# graph is being captured. Capture state is constant for this forward.
|
||||
@@ -1153,39 +1208,7 @@ class CodePredictorWrapper(nn.Module):
|
||||
|
||||
# Autoregressive loop: predict layers 1..G-1
|
||||
for step in range(1, num_groups):
|
||||
graph_key: int | tuple[int, int] = padded_bsz
|
||||
seq_len = max_seq
|
||||
if self._prefix_reprefill_enabled:
|
||||
actual_seq_len = step + 1
|
||||
for prefix_seq_len in self._prefix_reprefill_seq_lens:
|
||||
if prefix_seq_len < actual_seq_len:
|
||||
continue
|
||||
prefix_key = (padded_bsz, prefix_seq_len)
|
||||
if prefix_key in self._bucket_pos_ids:
|
||||
graph_key = prefix_key
|
||||
seq_len = prefix_seq_len
|
||||
break
|
||||
pos_ids = self._bucket_pos_ids.get(graph_key)
|
||||
if pos_ids is None:
|
||||
pos_ids = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.long)
|
||||
.unsqueeze(0)
|
||||
.expand(padded_bsz, -1)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
# Use captured device graph if available, otherwise call compiled fn.
|
||||
device_graph_entry = self._device_graphs.get(graph_key)
|
||||
|
||||
# Let the outer graph record the regular forward during capture;
|
||||
# normal inference still uses the inner graph replay fast path.
|
||||
if device_graph_entry is not None and not is_npu_capturing:
|
||||
device_graph_entry[0].replay()
|
||||
hidden_out = device_graph_entry[1]
|
||||
else:
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
|
||||
|
||||
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
logits = self._predict_step_logits(proj_buf, bsz, padded_bsz, step, is_npu_capturing)
|
||||
|
||||
# Sample next code via Gumbel-max.
|
||||
#
|
||||
@@ -1216,12 +1239,8 @@ class CodePredictorWrapper(nn.Module):
|
||||
else:
|
||||
# "per_call" mode: temperature-scaled + top-k -> Gumbel-max
|
||||
if use_sampling:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
step_uniforms = sample_uniforms[:, step - 1, :] if sample_uniforms is not None else None
|
||||
code = self._sample_codes_gumbel(scaled, generator=sample_generator, uniforms=step_uniforms)
|
||||
code = self._sample_per_call(logits, inv_temperature, top_k, sample_generator, step_uniforms)
|
||||
else:
|
||||
code = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Frame-local KV reuse for the Qwen3-TTS residual codebooks.
|
||||
|
||||
The scratch buffer belongs to one predictor worker, just like its projection
|
||||
buffer. Calls are serialized on the worker's CUDA stream. Every frame starts
|
||||
by overwriting positions 0 and 1; later queries can only read that frame's
|
||||
written positions. No request state survives between frames.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
|
||||
CodePredictorAttention,
|
||||
CodePredictorBaseModel,
|
||||
CodePredictorDecoderLayer,
|
||||
_rotate_half,
|
||||
)
|
||||
|
||||
from .short_kv_attention import short_kv_attention
|
||||
|
||||
|
||||
def _attention_cached(
|
||||
attn: CodePredictorAttention,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
write_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Attend only the new positions, reusing earlier positions' keys and values.
|
||||
|
||||
``k_cache``/``v_cache`` are ``[B, num_kv_heads, max_seq, head_dim]``;
|
||||
the new positions' keys and values are written at ``write_positions``
|
||||
(``[n]``). The attention kernel masks keys after each query position,
|
||||
so unwritten slots are never read.
|
||||
"""
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
qkv = attn.qkv_proj(hidden_states)
|
||||
q_raw, k_raw, v_raw = attn._split_qkv(qkv)
|
||||
q = attn.q_norm(q_raw.view(bsz, seq_len, attn.num_heads, attn.head_dim)).transpose(1, 2)
|
||||
k = attn.k_norm(k_raw.view(bsz, seq_len, attn.num_kv_heads, attn.head_dim)).transpose(1, 2)
|
||||
v = v_raw.view(bsz, seq_len, attn.num_kv_heads, attn.head_dim).transpose(1, 2)
|
||||
|
||||
cos, sin = position_embeddings
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
q = (q * cos) + (_rotate_half(q) * sin)
|
||||
k = (k * cos) + (_rotate_half(k) * sin)
|
||||
k_cache.index_copy_(2, write_positions, k.to(k_cache.dtype))
|
||||
v_cache.index_copy_(2, write_positions, v.to(v_cache.dtype))
|
||||
attn_out = short_kv_attention(q, k_cache, v_cache, write_positions, attn.scaling)
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz, seq_len, -1)
|
||||
return attn.o_proj(attn_out)
|
||||
|
||||
|
||||
def _layer_cached(
|
||||
layer: CodePredictorDecoderLayer,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
write_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = layer.input_layernorm(hidden_states)
|
||||
hidden_states = _attention_cached(
|
||||
layer.self_attn, hidden_states, position_embeddings, k_cache, v_cache, write_positions
|
||||
)
|
||||
hidden_states = residual + hidden_states
|
||||
residual = hidden_states
|
||||
hidden_states = layer.post_attention_layernorm(hidden_states)
|
||||
hidden_states = layer.mlp(hidden_states)
|
||||
return residual + hidden_states
|
||||
|
||||
|
||||
def forward_cached(
|
||||
model: CodePredictorBaseModel,
|
||||
inputs_embeds: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
kv_caches: Sequence[tuple[torch.Tensor, torch.Tensor]],
|
||||
write_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""``forward`` for the new positions only (``position_ids`` ``[B, n]``), reusing earlier K/V."""
|
||||
hidden_states = inputs_embeds
|
||||
with torch.amp.autocast(inputs_embeds.device.type, enabled=False):
|
||||
position_embeddings = model.rotary_emb(hidden_states, position_ids)
|
||||
for layer, (k_cache, v_cache) in zip(model.layers, kv_caches):
|
||||
hidden_states = _layer_cached(layer, hidden_states, position_embeddings, k_cache, v_cache, write_positions)
|
||||
hidden_states = model.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FrameLocalKVCache:
|
||||
"""Non-module execution state; checkpoint parameters remain on the predictor."""
|
||||
|
||||
def __init__(self, model: CodePredictorBaseModel, max_batch: int) -> None:
|
||||
self.model = model
|
||||
weight = next(model.parameters())
|
||||
attn = model.layers[0].self_attn
|
||||
self.buffer = torch.zeros(
|
||||
len(model.layers),
|
||||
2,
|
||||
max_batch,
|
||||
attn.num_kv_heads,
|
||||
attn.max_seq,
|
||||
attn.head_dim,
|
||||
device=weight.device,
|
||||
dtype=weight.dtype,
|
||||
)
|
||||
self.positions = {
|
||||
step: torch.arange(0 if step == 1 else step, step + 1, device=weight.device)
|
||||
for step in range(1, attn.max_seq - 1)
|
||||
}
|
||||
# Batch buckets share compiled kernels; outer MTP graphs still capture
|
||||
# fixed addresses. This helper never creates nested CUDA graphs.
|
||||
self.forward = torch.compile(forward_cached, dynamic=True, options={"epilogue_fusion": False})
|
||||
|
||||
def __call__(self, proj_buf: torch.Tensor, batch: int, step: int) -> torch.Tensor:
|
||||
if batch > self.buffer.shape[2]:
|
||||
raise ValueError("Qwen3-TTS cached predictor batch exceeds its configured capacity")
|
||||
first = 0 if step == 1 else step
|
||||
caches = [(layer[0, :batch], layer[1, :batch]) for layer in self.buffer]
|
||||
positions = self.positions[step]
|
||||
return self.forward(
|
||||
self.model,
|
||||
proj_buf[:batch, first : step + 1],
|
||||
positions.unsqueeze(0).expand(batch, -1),
|
||||
caches,
|
||||
positions,
|
||||
)
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Top-k Gumbel sampling with the outer MTP graph's supplied uniforms."""
|
||||
|
||||
import torch
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sample(
|
||||
logits,
|
||||
uniforms,
|
||||
output,
|
||||
ls: tl.constexpr,
|
||||
us: tl.constexpr,
|
||||
lc: tl.constexpr,
|
||||
uc: tl.constexpr,
|
||||
vocab: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
topk_block: tl.constexpr,
|
||||
inv_temperature: tl.constexpr,
|
||||
block: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
idx = tl.arange(0, block)
|
||||
value = tl.load(logits + row * ls + idx * lc, idx < vocab, other=-float("inf"))
|
||||
scaled = (value.to(tl.float32) * inv_temperature).to(value.dtype).to(tl.float32)
|
||||
if top_k > 0:
|
||||
if top_k == 1:
|
||||
threshold = tl.max(scaled, 0)
|
||||
else:
|
||||
ordered = tl.topk(scaled, topk_block)
|
||||
threshold = tl.sum(tl.where(tl.arange(0, topk_block) == top_k - 1, ordered, 0))
|
||||
scaled = tl.where(scaled < threshold, -float("inf"), scaled)
|
||||
u = tl.load(uniforms + row * us + idx * uc, idx < vocab, other=0.5)
|
||||
score = scaled - tldevice.log(-tldevice.log(u))
|
||||
score = tl.where(idx < vocab, score, -float("inf"))
|
||||
max_score = tl.max(score, 0)
|
||||
chosen = tl.min(tl.where(score == max_score, idx, block), 0)
|
||||
first_nan = tl.min(tl.where(score != score, idx, block), 0)
|
||||
tl.store(output + row, tl.where(first_nan < block, first_nan, chosen))
|
||||
|
||||
|
||||
def sample_code_topk_gumbel(
|
||||
logits: torch.Tensor, uniforms: torch.Tensor, top_k: int, inv_temperature: float
|
||||
) -> torch.Tensor:
|
||||
"""Match BF16 scale rounding, top-k ties, and first-index Gumbel argmax.
|
||||
|
||||
Random values are supplied by the outer graph, so RNG state and seeded
|
||||
per-request streams are unchanged. CUDA libdevice logs match ATen's logs.
|
||||
"""
|
||||
if logits.shape != uniforms.shape:
|
||||
raise ValueError("sampling uniforms must match logits shape")
|
||||
if logits.device != uniforms.device or not logits.is_cuda:
|
||||
raise ValueError("logits and uniforms must be on the same CUDA device")
|
||||
if top_k > logits.shape[-1]:
|
||||
raise ValueError("top_k must not exceed the vocabulary size")
|
||||
b, v = logits.shape
|
||||
out = torch.empty((b, 1), device=logits.device, dtype=torch.int64)
|
||||
_sample[(b,)](
|
||||
logits,
|
||||
uniforms,
|
||||
out,
|
||||
logits.stride(0),
|
||||
uniforms.stride(0),
|
||||
logits.stride(1),
|
||||
uniforms.stride(1),
|
||||
v,
|
||||
top_k,
|
||||
triton.next_power_of_2(max(1, top_k)),
|
||||
inv_temperature,
|
||||
triton.next_power_of_2(v),
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Qwen3-TTS first-frame decode eligibility."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def talker_first_audio_enabled(vllm_config: Any) -> bool:
|
||||
"""First-frame delivery requires CUDA MRv2 streaming with an in-process Talker.
|
||||
|
||||
Unsupported runners retain the regular codec path. Prefix-cache replay
|
||||
and distributed Talkers likewise retain the regular path.
|
||||
"""
|
||||
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM as Predictor
|
||||
|
||||
extra = Predictor._stage_connector_extra_config(vllm_config)
|
||||
model = vllm_config.model_config
|
||||
parallel = vllm_config.parallel_config
|
||||
return (
|
||||
Predictor._parse_bool_config(extra.get("talker_first_audio"))
|
||||
and current_omni_platform.is_cuda()
|
||||
and torch.device(vllm_config.device_config.device).type == "cuda"
|
||||
and bool(getattr(model, "use_v2_model_runner", False))
|
||||
and bool(getattr(model, "async_chunk", False))
|
||||
and parallel.tensor_parallel_size == 1
|
||||
and parallel.pipeline_parallel_size == 1
|
||||
and parallel.distributed_executor_backend in (None, "uni")
|
||||
and not vllm_config.cache_config.enable_prefix_caching
|
||||
)
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""First codec frame -> PCM inside the Talker process.
|
||||
|
||||
A stream's first frame has no decoder context, so its waveform does not depend
|
||||
on the Code2Wav stage's streaming state. Decoding it where the frame is
|
||||
produced removes the stage hop from time to first audio. Code2Wav advances
|
||||
its causal state without emitting that delivered frame again. Only streams
|
||||
without reference-code context use this path.
|
||||
|
||||
The decode runs beside the Talker's next step, so its latency is set by how
|
||||
many dependent kernels it issues. A single frame with no context attends to
|
||||
itself only: each softmax is over one key and equals 1 exactly, so attention
|
||||
returns the value projection. The single-frame path therefore
|
||||
skips q/k, RoPE, masks and softmax (about two thirds of the kernels).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.cuda import CUDAGraph
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
|
||||
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
|
||||
from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Decoder
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SUBFOLDER = "speech_tokenizer"
|
||||
|
||||
|
||||
class Qwen3TTSFirstFrameDecoder(nn.Module):
|
||||
def __init__(self, model_path: str) -> None:
|
||||
super().__init__()
|
||||
self.model_path = model_path
|
||||
config = Qwen3TTSTokenizerV2Config.from_pretrained(model_path, subfolder=_SUBFOLDER)
|
||||
self.decoder = Qwen3TTSTokenizerV2Decoder._from_config(config.decoder_config)
|
||||
self.decoder.eval()
|
||||
self.num_quantizers = int(config.decoder_config.num_quantizers)
|
||||
self.sample_rate = int(config.output_sample_rate)
|
||||
self._graphs: dict[int, tuple[CUDAGraph, torch.Tensor, torch.Tensor]] = {}
|
||||
|
||||
def load(self, vllm_config: VllmConfig) -> set[str]:
|
||||
"""Load decoder weights exactly as Code2Wav does; returns the loaded parameter names."""
|
||||
loader = DefaultModelLoader(vllm_config.load_config)
|
||||
source = DefaultModelLoader.Source(
|
||||
model_or_path=self.model_path,
|
||||
revision=vllm_config.model_config.revision,
|
||||
subfolder=_SUBFOLDER,
|
||||
)
|
||||
loaded = AutoWeightsLoader(self).load_weights(
|
||||
loader._get_weights_iterator(source),
|
||||
mapper=WeightsMapper(orig_to_new_prefix={"encoder.": None}),
|
||||
)
|
||||
self.decoder.to(device=vllm_config.device_config.device, dtype=vllm_config.model_config.dtype)
|
||||
if hasattr(self.decoder, "precompute_snake_caches"):
|
||||
self.decoder.precompute_snake_caches()
|
||||
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM as Predictor
|
||||
|
||||
extra = Predictor._stage_connector_extra_config(vllm_config)
|
||||
if Predictor._parse_bool_config(extra.get("decode_time_major_conv")):
|
||||
self.decoder.enable_time_major_conv()
|
||||
return loaded
|
||||
|
||||
def _decode_single_frame(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""``codes`` [n, num_quantizers, 1] -> waveform; equals ``_decode_xvec_first_chunk``."""
|
||||
d = self.decoder
|
||||
hidden = d.pre_conv(d.quantizer.decode(codes)).transpose(1, 2)
|
||||
t = d.pre_transformer
|
||||
h = t.input_proj(hidden)
|
||||
for layer in t.layers[: t.config.num_hidden_layers]:
|
||||
attention = layer.self_attn
|
||||
values = attention.v_proj(layer.input_layernorm(h))
|
||||
if attention.num_key_value_groups != 1:
|
||||
values = values.reshape(*values.shape[:-1], -1, attention.head_dim)
|
||||
values = values.repeat_interleave(attention.num_key_value_groups, dim=-2).flatten(-2)
|
||||
h = h + layer.self_attn_layer_scale(attention.o_proj(values))
|
||||
h = h + layer.mlp_layer_scale(layer.mlp(layer.post_attention_layernorm(h)))
|
||||
return d._conv_decode(t.output_proj(t.norm(h))).clamp(min=-1, max=1)
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture(self, batch_sizes: Sequence[int] = (1, 2, 4, 8)) -> None:
|
||||
device = next(self.decoder.parameters()).device
|
||||
for batch_size in sorted(set(batch_sizes)):
|
||||
static_input = torch.zeros(batch_size, self.num_quantizers, 1, dtype=torch.long, device=device)
|
||||
self._decode_single_frame(static_input)
|
||||
torch.accelerator.synchronize(device)
|
||||
graph = CUDAGraph()
|
||||
# A separate pool prevents overlap with Talker/MTP graph allocations.
|
||||
with torch.cuda.graph(graph, pool=torch.cuda.graph_pool_handle()):
|
||||
static_output = self._decode_single_frame(static_input)
|
||||
self._graphs[batch_size] = (graph, static_input, static_output)
|
||||
logger.info("Captured Talker first-frame decoder graphs for batch sizes %s", sorted(self._graphs))
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""``codes`` [n, num_quantizers] long on device -> float32 PCM [n, samples] (new tensor)."""
|
||||
if not self._graphs:
|
||||
return self._decode_single_frame(codes.unsqueeze(-1))[:, 0, :].float()
|
||||
outputs = []
|
||||
largest = max(self._graphs)
|
||||
for start in range(0, int(codes.shape[0]), largest):
|
||||
chunk = codes[start : start + largest]
|
||||
rows = int(chunk.shape[0])
|
||||
batch_size = min(size for size in self._graphs if size >= rows)
|
||||
graph, static_input, static_output = self._graphs[batch_size]
|
||||
static_input.zero_()
|
||||
static_input[:rows, :, 0].copy_(chunk)
|
||||
graph.replay()
|
||||
outputs.append(static_output[:rows, 0, :].to(dtype=torch.float32, copy=True))
|
||||
return torch.cat(outputs, dim=0)
|
||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
import os
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -16,6 +17,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
|
||||
from vllm_omni.data_entry_keys import FIRST_AUDIO_REQUIRED_KEY
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import parse_chunk_ramp
|
||||
|
||||
@@ -112,6 +114,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
dec_config = tok_config.decoder_config
|
||||
self.decoder = Qwen3TTSTokenizerV2Decoder._from_config(dec_config)
|
||||
self.decoder.eval()
|
||||
|
||||
self._num_quantizers = int(dec_config.num_quantizers)
|
||||
self._output_sample_rate = int(tok_config.output_sample_rate)
|
||||
self._total_upsample = int(self.decoder.total_upsample)
|
||||
@@ -302,6 +305,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
valid_indices: list[int] = []
|
||||
left_context_size = [0] * len(request_ids_list)
|
||||
ref_context_size = [0] * len(request_ids_list)
|
||||
first_audio_flags = [False] * len(request_ids_list)
|
||||
segment_finished_flags = [False] * len(request_ids_list)
|
||||
request_state_ids: list[str | None] = [None] * len(request_ids_list)
|
||||
ref_context_request_ids: list[str | None] = [None] * len(request_ids_list)
|
||||
@@ -336,6 +340,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
meta = info.get("meta", {})
|
||||
first_audio_flags[i] = _meta_bool(meta.get("first_audio", False))
|
||||
if "is_segment_finished" in meta:
|
||||
segment_finished_flags[i] = _meta_bool(meta["is_segment_finished"])
|
||||
if "left_context_size" in meta:
|
||||
@@ -404,6 +409,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
)
|
||||
state = {}
|
||||
self._decoder_state_cache[state_req_id] = state
|
||||
state.setdefault("skip_first_audio", first_audio_flags[i])
|
||||
state.setdefault("prefix_frames", 0)
|
||||
if state_req_id.startswith(_DUMMY_REQUEST_ID):
|
||||
state["_is_dummy_run"] = True
|
||||
@@ -515,6 +521,10 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
if self._batch_stats_log_every > 0 and self._batch_stats_forwards % self._batch_stats_log_every == 0:
|
||||
self.log_decode_batch_stats()
|
||||
|
||||
first_audio_required = [torch.tensor(False)] * num_req
|
||||
if request_states is not None:
|
||||
for row, idx in enumerate(valid_indices):
|
||||
first_audio_required[idx] = torch.tensor(bool(request_states[row].get("skip_first_audio", False)))
|
||||
audios: list[torch.Tensor] = [empty] * num_req
|
||||
srs = [sr_tensor] * num_req
|
||||
|
||||
@@ -538,7 +548,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audios, "sr": srs},
|
||||
multimodal_outputs={"model_outputs": audios, "sr": srs, FIRST_AUDIO_REQUIRED_KEY: first_audio_required},
|
||||
)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
|
||||
@@ -714,6 +724,11 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config decode_batch_max_size={decode_batch_max_size}")
|
||||
self._decode_batch_max_size = decode_batch_max_size
|
||||
decode_enable_tf32 = _get_bool_config("decode_enable_tf32", False)
|
||||
decode_time_major_conv = _get_bool_config("decode_time_major_conv", False)
|
||||
self.decoder.capture_first_audio_state_only = self._async_chunk and _get_bool_config(
|
||||
"talker_first_audio", False
|
||||
)
|
||||
decode_cudnn_benchmark = _get_bool_config("decode_cudnn_benchmark", False)
|
||||
else:
|
||||
codec_chunk_frames = 0
|
||||
codec_left_context_frames = 0
|
||||
@@ -722,6 +737,8 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
decode_cudagraph_batch_sizes = None
|
||||
decode_cudagraph_capture_sizes = None
|
||||
decode_enable_tf32 = False
|
||||
decode_cudnn_benchmark = False
|
||||
decode_time_major_conv = False
|
||||
|
||||
if decode_enable_tf32 and device.type == "cuda":
|
||||
# PyTorch exposes TF32 controls as process-wide CUDA backend
|
||||
@@ -738,21 +755,39 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
torch.get_float32_matmul_precision(),
|
||||
)
|
||||
|
||||
if decode_time_major_conv and device.type == "cuda":
|
||||
self.decoder.enable_time_major_conv()
|
||||
|
||||
self.decoder._initial_codec_chunk_frames = initial_codec_chunk_frames
|
||||
self.decoder._incremental_chunk_frames = codec_chunk_frames or 25
|
||||
self.decoder._incremental_chunk_ramp = list(codec_chunk_ramp or ())
|
||||
|
||||
if hasattr(self.decoder, "enable_cudagraph") and device.type == "cuda":
|
||||
try:
|
||||
self._maybe_enable_decoder_cudagraph(
|
||||
device=device,
|
||||
codec_chunk_frames=codec_chunk_frames,
|
||||
codec_left_context_frames=codec_left_context_frames,
|
||||
initial_codec_chunk_frames=initial_codec_chunk_frames,
|
||||
codec_chunk_ramp=codec_chunk_ramp,
|
||||
decode_cudagraph_batch_sizes=decode_cudagraph_batch_sizes,
|
||||
decode_cudagraph_capture_sizes=decode_cudagraph_capture_sizes,
|
||||
# Autotune only during warmup/capture, then restore the process
|
||||
# flags. The captured convolution algorithms remain in the
|
||||
# graphs without changing later models' cuDNN policy.
|
||||
autotune = (
|
||||
torch.backends.cudnn.flags(
|
||||
enabled=torch.backends.cudnn.enabled,
|
||||
benchmark=True,
|
||||
benchmark_limit=10,
|
||||
deterministic=torch.backends.cudnn.deterministic,
|
||||
allow_tf32=torch.backends.cudnn.allow_tf32,
|
||||
)
|
||||
if decode_cudnn_benchmark
|
||||
else nullcontext()
|
||||
)
|
||||
with autotune:
|
||||
self._maybe_enable_decoder_cudagraph(
|
||||
device=device,
|
||||
codec_chunk_frames=codec_chunk_frames,
|
||||
codec_left_context_frames=codec_left_context_frames,
|
||||
initial_codec_chunk_frames=initial_codec_chunk_frames,
|
||||
codec_chunk_ramp=codec_chunk_ramp,
|
||||
decode_cudagraph_batch_sizes=decode_cudagraph_batch_sizes,
|
||||
decode_cudagraph_capture_sizes=decode_cudagraph_capture_sizes,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to enable CUDA Graph for Code2Wav decoder",
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
"""Qwen3-TTS Code Predictor -- thin wrapper over CodePredictorWrapper."""
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Qwen3-TTS predictor: per-call sampling and opt-in CUDA frame-local KV reuse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.common.qwen3_code_predictor import (
|
||||
CodePredictorBaseModel,
|
||||
@@ -17,6 +22,11 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .cached_code_predictor import FrameLocalKVCache
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Backward-compat alias used by tests
|
||||
Qwen3TTSTalkerCodePredictorModelVLLM = CodePredictorBaseModel
|
||||
|
||||
@@ -48,6 +58,82 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(CodePredictorWrapp
|
||||
# Store talker_config for backward compat (accessed by some callers)
|
||||
self.talker_config = talker_config
|
||||
self._vllm_config = vllm_config
|
||||
extra = self._stage_connector_extra_config(vllm_config)
|
||||
self._kv_requested = self._parse_bool_config(extra.get("code_predictor_kv_cache"))
|
||||
self._fused_sampling = self._parse_bool_config(extra.get("code_predictor_fused_sampling"))
|
||||
self._frame_cache: FrameLocalKVCache | None = None
|
||||
|
||||
def _setup_compile(self) -> None:
|
||||
if self._compiled_model_fwd is not None:
|
||||
return
|
||||
weight = next(self.model.parameters())
|
||||
supported = (
|
||||
current_omni_platform.is_cuda()
|
||||
and weight.device.type == "cuda"
|
||||
and weight.dtype == torch.bfloat16
|
||||
and current_omni_platform.supports_torch_inductor()
|
||||
)
|
||||
if not self._kv_requested or not supported:
|
||||
if self._kv_requested:
|
||||
logger.warning("Qwen3-TTS frame-local KV requires CUDA BF16; using re-prefill")
|
||||
return super()._setup_compile()
|
||||
|
||||
# Import CUDA-only kernels only when this model explicitly opts in.
|
||||
from .cached_code_predictor import FrameLocalKVCache
|
||||
|
||||
self._model_dtype = weight.dtype
|
||||
self._lm_heads_list = list(self.lm_head)
|
||||
self._codec_embeds_list = list(self.model.codec_embedding)
|
||||
self._bucket_sizes = self._batch_bucket_sizes()
|
||||
max_batch = max(self._bucket_sizes)
|
||||
self._ensure_buffers(weight.device, weight.dtype, max_batch)
|
||||
cache = FrameLocalKVCache(self.model, max_batch)
|
||||
with torch._dynamo.config.patch(
|
||||
cache_size_limit=max(torch._dynamo.config.cache_size_limit, 2 * len(self._bucket_sizes))
|
||||
):
|
||||
for batch in self._bucket_sizes:
|
||||
for step in range(1, min(3, self._num_groups)):
|
||||
for _ in range(2):
|
||||
cache(self._proj_buf, batch, step)
|
||||
self._synchronize_warmup(weight.device)
|
||||
self._frame_cache = cache
|
||||
# Preserve the shared wrapper's initialization sentinel. Its full
|
||||
# re-prefill callable is unused while _frame_cache is active.
|
||||
self._compiled_model_fwd = self.model.forward
|
||||
logger.info("Qwen3-TTS frame-local KV warmed for batches %s", self._bucket_sizes)
|
||||
|
||||
def _predict_step_logits(
|
||||
self,
|
||||
proj_buf: torch.Tensor,
|
||||
bsz: int,
|
||||
padded_bsz: int,
|
||||
step: int,
|
||||
is_npu_capturing: bool,
|
||||
) -> torch.Tensor:
|
||||
if self._frame_cache is None:
|
||||
return super()._predict_step_logits(proj_buf, bsz, padded_bsz, step, is_npu_capturing)
|
||||
hidden = self._frame_cache(proj_buf, padded_bsz, step)
|
||||
return self._lm_heads_list[step - 1](hidden[:bsz, -1])
|
||||
|
||||
def _sample_per_call(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
inv_temperature: float,
|
||||
top_k: int,
|
||||
generator: torch.Generator | Sequence[torch.Generator | None] | None,
|
||||
uniforms: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
self._fused_sampling
|
||||
and current_omni_platform.is_cuda()
|
||||
and logits.is_cuda
|
||||
and uniforms is not None
|
||||
and uniforms.dtype == torch.float32
|
||||
):
|
||||
from .code_predictor_sampling import sample_code_topk_gumbel
|
||||
|
||||
return sample_code_topk_gumbel(logits, uniforms, top_k, inv_temperature)
|
||||
return super()._sample_per_call(logits, inv_temperature, top_k, generator, uniforms)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with vllm config context (required for VocabParallelEmbedding)."""
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm_omni.utils.speaker_cache import (
|
||||
from vllm_omni.worker.sampling_utils import get_tts_local_seed
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig
|
||||
from .first_audio import talker_first_audio_enabled
|
||||
from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder, resolve_x_vector_only
|
||||
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM
|
||||
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
|
||||
@@ -361,6 +362,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
}
|
||||
)
|
||||
|
||||
# Set by the MRV2 model state once eager frames are enabled.
|
||||
eager_frames_active = False
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
@@ -404,6 +408,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
# per-request additional_information.
|
||||
self.talker_mtp_output_key = ("codes", "audio")
|
||||
self.talker_mtp_graph_safe = True
|
||||
# A frame's residual codebooks depend only on the Talker hidden and
|
||||
# CB0 of the step that sampled it, so MRV2 may complete the frame at
|
||||
# the end of that step instead of in the next step's preprocess.
|
||||
self.mtp_eager_frames = talker_first_audio_enabled(vllm_config)
|
||||
# The runners bypass only the outer whole-MTP graph when explicit
|
||||
# generators are present, so seeded requests can still share one raw
|
||||
# batched MTP call with independent per-row streams.
|
||||
@@ -722,6 +730,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
logger.warning_once("runtime_additional_information is deprecated, use model_intermediate_buffer")
|
||||
model_config = getattr(getattr(self, "vllm_config", None), "model_config", None)
|
||||
async_chunk = bool(getattr(model_config, "async_chunk", False))
|
||||
spans = kwargs.get("request_token_spans")
|
||||
if async_chunk and self.eager_frames_active and spans is not None:
|
||||
return self._make_eager_omni_output(hidden, info_dicts, spans)
|
||||
audio_codes_list: list[torch.Tensor] = []
|
||||
ref_code_len_segments: list[tuple[int | torch.Tensor, int, torch.device]] = []
|
||||
ref_code_list: list[torch.Tensor] = []
|
||||
@@ -863,6 +874,44 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
mm.setdefault("meta", {})["codec_frame_valid"] = codec_frame_valid[:span_len]
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
|
||||
|
||||
def _make_eager_omni_output(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
info_dicts: list[Any],
|
||||
spans: list[tuple[int, int]],
|
||||
) -> OmniOutput:
|
||||
"""Async-chunk output when the runner fills frames after sampling.
|
||||
|
||||
Every row that sampled this step gets its codes and validity from the
|
||||
runner's eager MTP; the rest (non-final prefill chunks) carry no frame,
|
||||
so the per-request buffer rows need not be gathered here.
|
||||
"""
|
||||
num_tokens = spans[-1][1] if spans else 0
|
||||
q = int(self.talker_config.num_code_groups)
|
||||
mm: OmniPayload = {
|
||||
"codes": {"audio": torch.zeros((num_tokens, q), dtype=torch.long, device=hidden.device)},
|
||||
"meta": {
|
||||
"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8, device=hidden.device),
|
||||
"first_audio": torch.zeros((num_tokens,), dtype=torch.int8, device=hidden.device),
|
||||
},
|
||||
}
|
||||
ref_rows = [
|
||||
index
|
||||
for index, info in enumerate(info_dicts)
|
||||
if isinstance(info, dict)
|
||||
and isinstance(ref := info.get("codes", {}).get("ref"), torch.Tensor)
|
||||
and ref.numel() > 0
|
||||
and _should_publish_async_ref_codes(info)
|
||||
]
|
||||
if ref_rows:
|
||||
empty = torch.empty(0, dtype=torch.long)
|
||||
ref_code_list = [empty] * len(info_dicts)
|
||||
for index in ref_rows:
|
||||
ref_code_list[index] = info_dicts[index]["codes"]["ref"]
|
||||
_mark_async_ref_codes_published(info_dicts[index])
|
||||
mm["codes"]["ref"] = ref_code_list
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
|
||||
|
||||
# -------------------- preprocess / postprocess --------------------
|
||||
|
||||
def preprocess(
|
||||
@@ -1087,7 +1136,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
f"preprocess_decode_batch expected {len(req_infos)} input ids, got {int(input_ids_flat.numel())}"
|
||||
)
|
||||
|
||||
codec_frame_valid_batch = (input_ids_flat >= 0) & (input_ids_flat < self._codebook_vocab_size)
|
||||
eager = self.eager_frames_active
|
||||
codec_frame_valid_batch = (
|
||||
None if eager else (input_ids_flat >= 0) & (input_ids_flat < self._codebook_vocab_size)
|
||||
)
|
||||
device = input_ids_flat.device
|
||||
dtype = self._embedding_dtype
|
||||
# Request-independent constant (see :meth:`_init_runtime_buffers`) —
|
||||
@@ -1150,28 +1202,29 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
text_step = tts_pad_embed
|
||||
next_text_offset = text_offset
|
||||
|
||||
last_hidden = hs.get("last")
|
||||
if isinstance(last_hidden, torch.Tensor):
|
||||
past_hidden = last_hidden.to(device=device, dtype=dtype).reshape(1, -1)
|
||||
else:
|
||||
# Match scalar preprocess(): EOS / async-scheduling races can
|
||||
# arrive before postprocess has written last; zeros are filtered
|
||||
# downstream the same way as the single-request path.
|
||||
logger.warning_once(
|
||||
"Qwen3-TTS batched decode missing last hidden; zero-filling past_hidden "
|
||||
"to match scalar preprocess()"
|
||||
)
|
||||
past_hidden = torch.zeros_like(text_step)
|
||||
past_hidden_list.append(past_hidden)
|
||||
text_step_list.append(text_step)
|
||||
|
||||
info_update: dict[str, Any] = {
|
||||
"meta": {
|
||||
"talker_text_offset": int(next_text_offset),
|
||||
"codec_streaming": codec_streaming,
|
||||
"codec_frame_valid": codec_frame_valid_batch[len(updates)].reshape(()),
|
||||
},
|
||||
}
|
||||
if not eager:
|
||||
# Eager frames finished MTP last step: no hidden input or row validity needed.
|
||||
last_hidden = hs.get("last")
|
||||
if isinstance(last_hidden, torch.Tensor):
|
||||
past_hidden = last_hidden.to(device=device, dtype=dtype).reshape(1, -1)
|
||||
else:
|
||||
# Match scalar preprocess(): EOS / async-scheduling races can
|
||||
# arrive before postprocess has written last; zeros are filtered
|
||||
# downstream the same way as the single-request path.
|
||||
logger.warning_once(
|
||||
"Qwen3-TTS batched decode missing last hidden; zero-filling past_hidden "
|
||||
"to match scalar preprocess()"
|
||||
)
|
||||
past_hidden = torch.zeros_like(text_step)
|
||||
past_hidden_list.append(past_hidden)
|
||||
info_update["meta"]["codec_frame_valid"] = codec_frame_valid_batch[len(updates)].reshape(())
|
||||
if trailing_text_update is not None:
|
||||
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
|
||||
updates.append(info_update)
|
||||
@@ -1179,11 +1232,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
if input_embeds is None:
|
||||
input_embeds = self.embed_input_ids(input_ids_flat.reshape(-1, 1).to(torch.long))
|
||||
inputs_embeds_out = input_embeds.to(device=device, dtype=dtype).reshape(len(req_infos), -1)
|
||||
text_steps = torch.cat(text_step_list, dim=0)
|
||||
return (
|
||||
input_ids_flat,
|
||||
inputs_embeds_out,
|
||||
torch.cat(past_hidden_list, dim=0),
|
||||
torch.cat(text_step_list, dim=0),
|
||||
text_steps if eager else torch.cat(past_hidden_list, dim=0),
|
||||
text_steps,
|
||||
updates,
|
||||
)
|
||||
|
||||
@@ -1382,6 +1436,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
logger.info("Loaded %d weights for Qwen3TTSTalkerForConditionalGeneration", len(loaded))
|
||||
self._build_stacked_codec_embed()
|
||||
if talker_first_audio_enabled(self.vllm_config):
|
||||
from .first_frame_decoder import Qwen3TTSFirstFrameDecoder
|
||||
|
||||
self.first_frame_decoder = Qwen3TTSFirstFrameDecoder(self.model_path)
|
||||
decoder_loaded = self.first_frame_decoder.load(self.vllm_config)
|
||||
loaded = set(loaded) | {f"first_frame_decoder.{name}" for name in decoder_loaded}
|
||||
return loaded
|
||||
|
||||
def _build_stacked_codec_embed(self) -> None:
|
||||
@@ -1569,6 +1629,35 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
mtp = talker_mtp
|
||||
get_mtp_seed = staticmethod(get_tts_local_seed)
|
||||
|
||||
# Postprocess only stores the last hidden for the deferred decode MTP.
|
||||
eager_frames_need_postprocess = False
|
||||
|
||||
def eager_decode_settled(self, info: dict[str, Any]) -> bool:
|
||||
"""Decode preprocess is a no-op for this row: its trailing text is spent.
|
||||
|
||||
From then on every decode step feeds the frame embedding plus the pad
|
||||
text step and changes no per-request state (the non-streaming
|
||||
CustomVoice/VoiceDesign case after the first decode).
|
||||
"""
|
||||
hs = info.get("hidden_states")
|
||||
tail = hs.get("trailing_text") if isinstance(hs, dict) else None
|
||||
meta = info.get("meta")
|
||||
offset = meta.get("talker_text_offset", 0) if isinstance(meta, dict) else 0
|
||||
return isinstance(tail, torch.Tensor) and tail.numel() == 0 and not offset
|
||||
|
||||
def eager_settled_text_step(self) -> torch.Tensor:
|
||||
return self._tts_pad_embed.reshape(1, -1)
|
||||
|
||||
def capture_first_frame_graphs(self) -> None:
|
||||
# Set by load_weights when talker_first_audio_enabled accepts the connector option.
|
||||
decoder = getattr(self, "first_frame_decoder", None)
|
||||
if decoder is not None:
|
||||
decoder.capture()
|
||||
|
||||
def mtp_frame_valid(self, layer0: torch.Tensor) -> torch.Tensor:
|
||||
"""Rows whose CB0 is a codec id rather than EOS/special; matches decode preprocess."""
|
||||
return (layer0 >= 0) & (layer0 < self._codebook_vocab_size)
|
||||
|
||||
@property
|
||||
def mtp_output_key(self) -> tuple[str, str]:
|
||||
return self.talker_mtp_output_key
|
||||
|
||||
@@ -71,6 +71,7 @@ class CUDAGraphDecoderWrapper:
|
||||
self.combined_states: dict[tuple[str, int, int], dict] = {}
|
||||
self.icl_prefix_states: dict[int, dict] = {}
|
||||
self.xvec_prefix_states: dict[int, dict] = {}
|
||||
self.xvec_prefix_state_only_states: dict[int, dict] = {}
|
||||
self.stateless_states: dict[tuple[int, int], dict] = {}
|
||||
|
||||
self._device = None
|
||||
@@ -230,6 +231,7 @@ class CUDAGraphDecoderWrapper:
|
||||
*self._retained_kv_caches(self.combined_states),
|
||||
*self._retained_kv_caches(self.icl_prefix_states),
|
||||
*self._retained_kv_caches(self.xvec_prefix_states),
|
||||
*self._retained_kv_caches(self.xvec_prefix_state_only_states),
|
||||
]
|
||||
batched_captures = sum(rows for rows, _ in retained)
|
||||
|
||||
@@ -248,7 +250,7 @@ class CUDAGraphDecoderWrapper:
|
||||
rows_reason=(
|
||||
f"rows across {len(retained)} retained caches in "
|
||||
f"{len(self.combined_states)} suffix / {len(self.icl_prefix_states)} icl-prefix / "
|
||||
f"{len(self.xvec_prefix_states)} xvec-prefix captures"
|
||||
f"{len(self.xvec_prefix_states) + len(self.xvec_prefix_state_only_states)} xvec-prefix captures"
|
||||
),
|
||||
allocation_note=(
|
||||
"counted from caches transformers has materialized; captures whose tensors are still "
|
||||
@@ -424,6 +426,14 @@ class CUDAGraphDecoderWrapper:
|
||||
logger.info("Captured xvec prefix CUDA Graph for batch=%d", batch_size)
|
||||
except Exception:
|
||||
logger.warning("Failed to capture xvec prefix CUDA Graph for batch=%d", batch_size, exc_info=True)
|
||||
if getattr(self.decoder, "capture_first_audio_state_only", False) and self.initial_chunk_frames == 1:
|
||||
try:
|
||||
self._capture_xvec_prefix(batch_size, device, dtype, state_only=True)
|
||||
logger.info("Captured xvec state-only CUDA Graph for batch=%d", batch_size)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to capture xvec state-only graph for batch=%d", batch_size, exc_info=True
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Starting CUDA Graph warmup for %d shapes: batch_sizes=%s seq_lens=%s",
|
||||
@@ -631,7 +641,9 @@ class CUDAGraphDecoderWrapper:
|
||||
"cache": caches,
|
||||
}
|
||||
|
||||
def _capture_xvec_prefix(self, batch_size: int, device: torch.device, dtype: torch.dtype) -> None:
|
||||
def _capture_xvec_prefix(
|
||||
self, batch_size: int, device: torch.device, dtype: torch.dtype, *, state_only: bool = False
|
||||
) -> None:
|
||||
static_input = torch.zeros(
|
||||
batch_size,
|
||||
self.num_quantizers,
|
||||
@@ -639,8 +651,13 @@ class CUDAGraphDecoderWrapper:
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
# The Talker delivers exactly one frame upstream: a one-frame first
|
||||
# chunk then only needs its state; a longer one keeps the other frames.
|
||||
first_chunk = (
|
||||
self.decoder._decode_xvec_first_chunk_state_only if state_only else self.decoder._decode_xvec_first_chunk
|
||||
)
|
||||
with torch.no_grad():
|
||||
_ = self.decoder._decode_xvec_first_chunk(static_input, {})
|
||||
_ = first_chunk(static_input, {})
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
config = self.decoder.config
|
||||
@@ -662,13 +679,15 @@ class CUDAGraphDecoderWrapper:
|
||||
graph = CUDAGraph()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_output = self.decoder._decode_xvec_first_chunk(static_input, caches)
|
||||
static_output = first_chunk(static_input, caches)
|
||||
|
||||
self.xvec_prefix_states[batch_size] = {
|
||||
states = self.xvec_prefix_state_only_states if state_only else self.xvec_prefix_states
|
||||
states[batch_size] = {
|
||||
"graph": graph,
|
||||
"input": {"codes": static_input},
|
||||
"output": static_output,
|
||||
"cache": caches,
|
||||
"state_only": state_only,
|
||||
}
|
||||
|
||||
def _ensure_suffix_buffers(self, caches: dict) -> None:
|
||||
@@ -808,7 +827,14 @@ class CUDAGraphDecoderWrapper:
|
||||
codes_list: list[torch.Tensor],
|
||||
request_caches: list[dict],
|
||||
) -> list[torch.Tensor] | None:
|
||||
available = set(getattr(self, "xvec_prefix_states", {}))
|
||||
skip_flags = [bool(cache.get("skip_first_audio", False)) for cache in request_caches]
|
||||
# Full PCM graphs remain available even when the Talker cannot use
|
||||
# direct delivery (or only some requests qualify). The option is only
|
||||
# a capture hint; request metadata decides which audio is still owed.
|
||||
states = getattr(self, "xvec_prefix_states", {})
|
||||
if skip_flags and all(skip_flags):
|
||||
states = getattr(self, "xvec_prefix_state_only_states", {}) or states
|
||||
available = set(states)
|
||||
if not available:
|
||||
self._record_graph_fallback("xvec_prefix:no_graph", len(codes_list))
|
||||
return None
|
||||
@@ -835,7 +861,7 @@ class CUDAGraphDecoderWrapper:
|
||||
self._record_graph_fallback("xvec_prefix:no_graph", len(codes_list))
|
||||
return None
|
||||
|
||||
state = self.xvec_prefix_states[batch_size]
|
||||
state = states[batch_size]
|
||||
static_input = state["input"]["codes"]
|
||||
static_input.zero_()
|
||||
for row, codes in enumerate(codes_list):
|
||||
@@ -855,7 +881,12 @@ class CUDAGraphDecoderWrapper:
|
||||
cache["suffix_conv"] = static_caches["suffix_conv"][row : row + 1].clone()
|
||||
cache["suffix_frames"] = self.initial_chunk_frames
|
||||
self._ensure_suffix_buffers(cache)
|
||||
outputs.append(state["output"][row : row + 1].clone())
|
||||
if state.get("state_only"):
|
||||
outputs.append(static_input.new_zeros((1, 1, 0), dtype=torch.float32))
|
||||
elif skip_flags[row]:
|
||||
outputs.append(state["output"][row : row + 1, :, self.decoder.total_upsample :].clone())
|
||||
else:
|
||||
outputs.append(state["output"][row : row + 1].clone())
|
||||
return outputs
|
||||
|
||||
def _decode_suffix_batch(
|
||||
@@ -983,6 +1014,8 @@ class CUDAGraphDecoderWrapper:
|
||||
def _decode_request_fallback(self, codes: torch.Tensor, cache: dict) -> torch.Tensor:
|
||||
if "suffix_quantized" not in cache:
|
||||
if int(cache["prefix_frames"]) == 0:
|
||||
if cache.get("skip_first_audio", False):
|
||||
return self.decoder._decode_stream_first_chunk(codes, cache)
|
||||
return self.decoder._decode_xvec_first_chunk(codes, cache)
|
||||
prefix_frames = int(cache["prefix_frames"])
|
||||
output = self.decoder._decode_icl_first_chunk(codes, cache, prefix_frames)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Short causal attention for the scratch KV cache inside a code predictor."""
|
||||
|
||||
import torch
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _short_kv_attention(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
positions_ptr,
|
||||
out_ptr,
|
||||
q_stride_b: tl.constexpr,
|
||||
q_stride_h: tl.constexpr,
|
||||
q_stride_n: tl.constexpr,
|
||||
k_stride_b: tl.constexpr,
|
||||
k_stride_h: tl.constexpr,
|
||||
k_stride_n: tl.constexpr,
|
||||
v_stride_b: tl.constexpr,
|
||||
v_stride_h: tl.constexpr,
|
||||
v_stride_n: tl.constexpr,
|
||||
heads: tl.constexpr,
|
||||
num_queries: tl.constexpr,
|
||||
num_keys: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
queries_per_kv: tl.constexpr,
|
||||
scale: tl.constexpr,
|
||||
block_n: tl.constexpr,
|
||||
block_d: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0)
|
||||
h = tl.program_id(1)
|
||||
qi = tl.program_id(2)
|
||||
kvh = h // queries_per_kv
|
||||
d = tl.arange(0, block_d)
|
||||
n = tl.arange(0, block_n)
|
||||
last = tl.load(positions_ptr + qi)
|
||||
q = tl.load(q_ptr + b * q_stride_b + h * q_stride_h + qi * q_stride_n + d, d < head_dim, other=0).to(tl.float32)
|
||||
k = tl.load(
|
||||
k_ptr + b * k_stride_b + kvh * k_stride_h + n[:, None] * k_stride_n + d[None, :],
|
||||
(n[:, None] < num_keys) & (n[:, None] <= last) & (d[None, :] < head_dim),
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
score = tl.sum(k * q[None, :], 1) * scale
|
||||
score = tl.where((n < num_keys) & (n <= last), score, -float("inf"))
|
||||
p = tl.exp(score - tl.max(score, 0))
|
||||
p = p / tl.sum(p, 0)
|
||||
v = tl.load(
|
||||
v_ptr + b * v_stride_b + kvh * v_stride_h + n[:, None] * v_stride_n + d[None, :],
|
||||
(n[:, None] < num_keys) & (n[:, None] <= last) & (d[None, :] < head_dim),
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
out = tl.sum(p[:, None] * v, 0)
|
||||
tl.store(out_ptr + ((b * num_queries + qi) * heads + h) * head_dim + d, out, d < head_dim)
|
||||
|
||||
|
||||
def short_kv_attention(
|
||||
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, positions: torch.Tensor, scale: float
|
||||
) -> torch.Tensor:
|
||||
"""One/two-query GQA over a frame-local scratch cache with contiguous head dimensions.
|
||||
|
||||
The returned [B, H, Q, D] tensor has a contiguous [B, Q, H, D] view,
|
||||
avoiding a separate transpose-copy before the output projection.
|
||||
"""
|
||||
b, h, nq, d = q.shape
|
||||
nk = k.shape[2]
|
||||
out = torch.empty((b, nq, h, d), device=q.device, dtype=q.dtype)
|
||||
_short_kv_attention[(b, h, nq)](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
positions,
|
||||
out,
|
||||
*q.stride()[:3],
|
||||
*k.stride()[:3],
|
||||
*v.stride()[:3],
|
||||
h,
|
||||
nq,
|
||||
nk,
|
||||
d,
|
||||
h // k.shape[1],
|
||||
scale,
|
||||
triton.next_power_of_2(nk),
|
||||
triton.next_power_of_2(d),
|
||||
num_warps=4,
|
||||
)
|
||||
return out.transpose(1, 2)
|
||||
+59
-39
@@ -997,6 +997,33 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
if count > 0:
|
||||
logger.info("Precomputed exp caches for %d SnakeBeta activations", count)
|
||||
|
||||
# Set by enable_time_major_conv(); a plain object so it stays out of state_dict().
|
||||
_time_major_stack = None
|
||||
|
||||
def enable_time_major_conv(self) -> None:
|
||||
"""Run the upsample + decoder convs time-major: one tensor-core GEMM per conv.
|
||||
|
||||
Call after the decoder reaches its device and dtype (the stack keeps
|
||||
GEMM-layout weight copies) and before CUDA graph capture.
|
||||
"""
|
||||
from .time_major_stack import TimeMajorConvStack
|
||||
|
||||
self._time_major_stack = TimeMajorConvStack(self)
|
||||
logger.info("Qwen3-TTS decoder: time-major (GEMM) conv stack enabled")
|
||||
|
||||
def _conv_decode(self, hidden: torch.Tensor) -> torch.Tensor:
|
||||
"""Pre-transformer output ``[B, T, latent]`` -> unclamped waveform ``[B, 1, T * total_upsample]``."""
|
||||
if self._time_major_stack is not None:
|
||||
return self._time_major_stack(hidden)
|
||||
hidden = hidden.permute(0, 2, 1)
|
||||
for blocks in self.upsample:
|
||||
for block in blocks:
|
||||
hidden = block(hidden)
|
||||
wav = hidden
|
||||
for block in self.decoder:
|
||||
wav = block(wav)
|
||||
return wav
|
||||
|
||||
def enable_cudagraph(
|
||||
self,
|
||||
capture_modes: tuple[str, ...] = ("icl", "xvec"),
|
||||
@@ -1056,15 +1083,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
hidden = self.pre_conv(hidden).transpose(1, 2)
|
||||
|
||||
hidden = self.pre_transformer(inputs_embeds=hidden).last_hidden_state
|
||||
hidden = hidden.permute(0, 2, 1)
|
||||
for blocks in self.upsample:
|
||||
for block in blocks:
|
||||
hidden = block(hidden)
|
||||
|
||||
wav = hidden
|
||||
for block in self.decoder:
|
||||
wav = block(wav)
|
||||
return wav.clamp(min=-1, max=1)
|
||||
return self._conv_decode(hidden).clamp(min=-1, max=1)
|
||||
|
||||
def _decode_icl_first_chunk(
|
||||
self,
|
||||
@@ -1126,20 +1145,32 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
caches.update({"past_key_values": prefix_cache})
|
||||
caches.update({"prefix_hidden": prefix_hidden})
|
||||
|
||||
hidden = hidden.permute(0, 2, 1)
|
||||
|
||||
for blocks in self.upsample:
|
||||
for block in blocks:
|
||||
hidden = block(hidden)
|
||||
wav = hidden
|
||||
|
||||
for block in self.decoder:
|
||||
wav = block(wav)
|
||||
wav = wav.clamp(min=-1, max=1)
|
||||
return wav
|
||||
return self._conv_decode(hidden).clamp(min=-1, max=1)
|
||||
|
||||
def _decode_xvec_first_chunk(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
|
||||
"""Initialize prefixless incremental state from the first codec chunk."""
|
||||
suffix_hidden = self._init_xvec_first_chunk_state(codes, caches)
|
||||
return self._conv_decode(suffix_hidden).clamp(min=-1, max=1)
|
||||
|
||||
def _decode_xvec_first_chunk_state_only(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
|
||||
"""State for later chunks when the first chunk's audio was delivered upstream."""
|
||||
return self._init_xvec_first_chunk_state(codes, caches)
|
||||
|
||||
# Capture hint only: the raw connector option does not guarantee that
|
||||
# a request received upstream audio. Only its skip_first_audio marker does.
|
||||
capture_first_audio_state_only = False
|
||||
|
||||
def _decode_stream_first_chunk(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
|
||||
if not caches.get("skip_first_audio", False):
|
||||
return self._decode_xvec_first_chunk(codes, caches)
|
||||
if int(codes.shape[-1]) <= 1:
|
||||
self._decode_xvec_first_chunk_state_only(codes, caches)
|
||||
return codes.new_zeros((codes.shape[0], 1, 0), dtype=torch.float32)
|
||||
# The Talker delivered only the first frame; a longer first chunk
|
||||
# (adaptive or ramped schedules) still owes the audio of the rest.
|
||||
return self._decode_xvec_first_chunk(codes, caches)[..., self.total_upsample :]
|
||||
|
||||
def _init_xvec_first_chunk_state(self, codes: torch.Tensor, caches: dict) -> torch.Tensor:
|
||||
hidden = self.quantizer.decode(codes)
|
||||
caches["decoder_prefix_frames"] = 0
|
||||
caches["ref_hidden"] = hidden[:, :, :0]
|
||||
@@ -1172,16 +1203,8 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
).last_hidden_state
|
||||
caches["past_key_values"] = empty_prefix_cache
|
||||
caches["prefix_hidden"] = suffix_hidden[:, :0, :]
|
||||
|
||||
hidden = suffix_hidden.permute(0, 2, 1)
|
||||
for blocks in self.upsample:
|
||||
for block in blocks:
|
||||
hidden = block(hidden)
|
||||
wav = hidden
|
||||
for block in self.decoder:
|
||||
wav = block(wav)
|
||||
caches["suffix_frames"] = int(codes.shape[-1])
|
||||
return wav.clamp(min=-1, max=1)
|
||||
return suffix_hidden
|
||||
|
||||
def _decode_suffix(
|
||||
self,
|
||||
@@ -1229,14 +1252,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
hidden = torch.cat([caches["prefix_hidden"], suffix_hidden], dim=1)
|
||||
|
||||
required_frames = min(hidden.shape[1], new_frames + _DOWNSTREAM_CONTEXT_FRAME)
|
||||
hidden = hidden[:, -required_frames:, :].permute(0, 2, 1)
|
||||
for blocks in self.upsample:
|
||||
for block in blocks:
|
||||
hidden = block(hidden)
|
||||
|
||||
wav = hidden
|
||||
for block in self.decoder:
|
||||
wav = block(wav)
|
||||
wav = self._conv_decode(hidden[:, -required_frames:, :])
|
||||
wav = wav[..., -new_frames * self.total_upsample :].clamp(min=-1, max=1)
|
||||
return wav, next_quantized, next_conv
|
||||
|
||||
@@ -1300,7 +1316,7 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
f"prefix_frames={prefix_frames}, total_frames={codes.shape[-1]}"
|
||||
)
|
||||
if prefix_frames == 0:
|
||||
return self._decode_xvec_first_chunk(codes, caches)
|
||||
return self._decode_stream_first_chunk(codes, caches)
|
||||
return self._decode_icl_first_chunk(codes, caches, prefix_frames)
|
||||
else:
|
||||
decoder_prefix_frames = int(caches.get("decoder_prefix_frames", prefix_frames))
|
||||
@@ -1546,7 +1562,11 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
return None
|
||||
batched_codes = torch.cat(codes_list, dim=0)
|
||||
batched_cache: dict[str, Any] = {}
|
||||
output = self._decode_xvec_first_chunk(batched_codes, batched_cache)
|
||||
skip_flags = {bool(cache.get("skip_first_audio", False)) for cache in request_caches}
|
||||
if len(skip_flags) != 1:
|
||||
return None
|
||||
batched_cache["skip_first_audio"] = skip_flags.pop()
|
||||
output = self._decode_stream_first_chunk(batched_codes, batched_cache)
|
||||
outputs: list[torch.Tensor] = []
|
||||
for row, cache in enumerate(request_caches):
|
||||
for key in ("ref_hidden", "ref_conv", "prefix_hidden", "suffix_quantized", "suffix_conv"):
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Time-major causal 1-D convolution for streaming vocoders.
|
||||
|
||||
Activations are ``[B, T, C]`` so that every dense convolution is one GEMM over
|
||||
``B * T`` rows (bf16 tensor cores through cuBLAS) instead of an NCT cuDNN
|
||||
convolution, which for these long, thin decoder layers picks non-tensor-core
|
||||
kernels and adds layout transforms around each call.
|
||||
|
||||
- :func:`snake_im2col` gathers each output step's causal, dilated window into a
|
||||
``[B * T, K * C]`` row (tap-major), optionally applying ``bias`` and SnakeBeta
|
||||
on the way in; ``K = 1`` is a standalone (bias +) SnakeBeta.
|
||||
- :func:`causal_conv` is the dense causal (dilated) convolution as an implicit
|
||||
GEMM: each K-block of the reduction reads the input rows shifted by one tap,
|
||||
so the ``K * C`` im2col matrix is never materialized; bias and an optional
|
||||
residual are added in the epilogue.
|
||||
- :func:`transposed_overlap_add` finishes a transposed convolution of kernel
|
||||
``stride`` or ``2 * stride`` whose GEMM produced ``[B, T, K, C_out]``: the
|
||||
overlap-add of consecutive inputs, the bias and the causal right trim.
|
||||
|
||||
Weights come from ``nn.Conv1d``/``nn.ConvTranspose1d`` via :func:`conv_gemm_weight`
|
||||
and :func:`transposed_gemm_weight`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
|
||||
def conv_gemm_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""``nn.Conv1d`` weight ``[C_out, C_in, K]`` -> tap-major GEMM weight ``[K * C_in, C_out]``."""
|
||||
c_out, c_in, k = weight.shape
|
||||
return weight.detach().permute(2, 1, 0).reshape(k * c_in, c_out).contiguous()
|
||||
|
||||
|
||||
def transposed_gemm_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""``nn.ConvTranspose1d`` weight ``[C_in, C_out, K]`` -> GEMM weight ``[C_in, K * C_out]``."""
|
||||
c_in, c_out, k = weight.shape
|
||||
return weight.detach().permute(0, 2, 1).reshape(c_in, k * c_out).contiguous()
|
||||
|
||||
|
||||
def snake_im2col_reference(
|
||||
x: torch.Tensor,
|
||||
kernel_size: int,
|
||||
dilation: int = 1,
|
||||
bias: torch.Tensor | None = None,
|
||||
snake: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Eager equivalent of :func:`snake_im2col` (also the CPU path)."""
|
||||
if bias is not None:
|
||||
x = x + bias
|
||||
if snake is not None:
|
||||
exp_alpha, inv_beta = snake
|
||||
x = x + (inv_beta.float() * torch.sin(x.float() * exp_alpha.float()).square()).to(x.dtype)
|
||||
if kernel_size == 1:
|
||||
return x.reshape(-1, x.shape[-1])
|
||||
t = x.shape[1]
|
||||
xp = torch.nn.functional.pad(x, (0, 0, (kernel_size - 1) * dilation, 0))
|
||||
cols = torch.cat([xp[:, j * dilation : j * dilation + t] for j in range(kernel_size)], dim=-1)
|
||||
return cols.reshape(-1, cols.shape[-1])
|
||||
|
||||
|
||||
def causal_conv_reference(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
kernel_size: int,
|
||||
dilation: int = 1,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Eager equivalent of :func:`causal_conv`: ``[B, T, C]`` -> ``[B, T, C_out]``."""
|
||||
bsz, t, _ = x.shape
|
||||
cols = snake_im2col_reference(x, kernel_size, dilation)
|
||||
out = cols @ weight if bias is None else torch.addmm(bias, cols, weight)
|
||||
out = out.view(bsz, t, -1)
|
||||
return out if residual is None else out + residual
|
||||
|
||||
|
||||
def transposed_overlap_add_reference(z: torch.Tensor, stride: int, bias: torch.Tensor) -> torch.Tensor:
|
||||
"""Eager equivalent of :func:`transposed_overlap_add`."""
|
||||
bsz, t, k, c_out = z.shape
|
||||
out = z[:, :, :stride]
|
||||
if k > stride:
|
||||
out = out + torch.nn.functional.pad(z[:, :-1, stride:], (0, 0, 0, 0, 1, 0))
|
||||
return out.reshape(bsz, t * stride, c_out) + bias
|
||||
|
||||
|
||||
if HAS_TRITON:
|
||||
|
||||
@triton.jit
|
||||
def _snake_im2col_kernel(
|
||||
x_ptr,
|
||||
bias_ptr,
|
||||
exp_alpha_ptr,
|
||||
inv_beta_ptr,
|
||||
out_ptr,
|
||||
n_rows,
|
||||
t_len,
|
||||
c_len,
|
||||
dilation,
|
||||
K: tl.constexpr, # noqa: N803
|
||||
HAS_BIAS: tl.constexpr, # noqa: N803
|
||||
HAS_SNAKE: tl.constexpr, # noqa: N803
|
||||
BLOCK_R: tl.constexpr, # noqa: N803
|
||||
BLOCK_C: tl.constexpr, # noqa: N803
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
|
||||
chans = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
|
||||
row_mask = rows < n_rows
|
||||
chan_mask = chans < c_len
|
||||
t = rows % t_len
|
||||
if HAS_BIAS:
|
||||
bias = tl.load(bias_ptr + chans, mask=chan_mask, other=0.0)
|
||||
if HAS_SNAKE:
|
||||
exp_alpha = tl.load(exp_alpha_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)
|
||||
inv_beta = tl.load(inv_beta_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)
|
||||
out_mask = row_mask[:, None] & chan_mask[None, :]
|
||||
for j in tl.static_range(K):
|
||||
shift = (K - 1 - j) * dilation
|
||||
valid = row_mask & (t >= shift)
|
||||
src = rows - shift
|
||||
x = tl.load(
|
||||
x_ptr + src[:, None].to(tl.int64) * c_len + chans[None, :],
|
||||
mask=valid[:, None] & chan_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
if HAS_BIAS:
|
||||
# Causal padding is applied to the conv input, after bias.
|
||||
x = tl.where(valid[:, None], x + bias[None, :], 0.0).to(x_ptr.dtype.element_ty)
|
||||
if HAS_SNAKE:
|
||||
s = tl.sin(x.to(tl.float32) * exp_alpha[None, :])
|
||||
x = x + (inv_beta[None, :] * s * s).to(x.dtype)
|
||||
tl.store(out_ptr + rows[:, None].to(tl.int64) * (K * c_len) + j * c_len + chans[None, :], x, mask=out_mask)
|
||||
|
||||
@triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK_M": bm, "BLOCK_N": bn, "BLOCK_K": bk}, num_warps=w, num_stages=st)
|
||||
for bm, bn, bk, w, st in (
|
||||
(128, 128, 64, 8, 3),
|
||||
(128, 64, 64, 4, 4),
|
||||
(64, 128, 64, 4, 4),
|
||||
(128, 32, 32, 4, 4),
|
||||
(64, 64, 32, 4, 4),
|
||||
(256, 32, 32, 8, 3),
|
||||
(256, 64, 32, 8, 3),
|
||||
(128, 16, 32, 4, 4),
|
||||
)
|
||||
],
|
||||
key=["row_bucket", "n_cols", "c_len", "K", "dilation"],
|
||||
)
|
||||
@triton.jit
|
||||
def _causal_conv_gemm_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
bias_ptr,
|
||||
res_ptr,
|
||||
out_ptr,
|
||||
n_rows,
|
||||
n_cols,
|
||||
c_len,
|
||||
t_len,
|
||||
dilation,
|
||||
row_bucket, # autotune key only: power-of-two bucket of n_rows
|
||||
K: tl.constexpr, # noqa: N803
|
||||
HAS_BIAS: tl.constexpr, # noqa: N803
|
||||
HAS_RES: tl.constexpr, # noqa: N803
|
||||
PRECISION: tl.constexpr, # noqa: N803
|
||||
BLOCK_M: tl.constexpr, # noqa: N803
|
||||
BLOCK_N: tl.constexpr, # noqa: N803
|
||||
BLOCK_K: tl.constexpr, # noqa: N803
|
||||
):
|
||||
# Row-major tiles, N fastest: neighbouring programs share the input rows.
|
||||
pid = tl.program_id(0)
|
||||
num_n = tl.cdiv(n_cols, BLOCK_N)
|
||||
pid_m = pid // num_n
|
||||
pid_n = pid % num_n
|
||||
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
row_mask = rows < n_rows
|
||||
col_mask = cols < n_cols
|
||||
t = rows % t_len
|
||||
k_blocks = tl.cdiv(c_len, BLOCK_K)
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for kk in range(0, K * k_blocks):
|
||||
tap = kk // k_blocks
|
||||
chans = (kk % k_blocks) * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
chan_mask = chans < c_len
|
||||
shift = (K - 1 - tap) * dilation
|
||||
a = tl.load(
|
||||
x_ptr + (rows - shift)[:, None] * c_len + chans[None, :],
|
||||
mask=(row_mask & (t >= shift))[:, None] & chan_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(
|
||||
w_ptr + (tap * c_len + chans)[:, None] * n_cols + cols[None, :],
|
||||
mask=chan_mask[:, None] & col_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
acc = tl.dot(a, b, acc, input_precision=PRECISION)
|
||||
if HAS_BIAS:
|
||||
acc += tl.load(bias_ptr + cols, mask=col_mask, other=0.0).to(tl.float32)[None, :]
|
||||
out_offsets = rows[:, None] * n_cols + cols[None, :]
|
||||
out_mask = row_mask[:, None] & col_mask[None, :]
|
||||
if HAS_RES:
|
||||
acc += tl.load(res_ptr + out_offsets, mask=out_mask, other=0.0).to(tl.float32)
|
||||
tl.store(out_ptr + out_offsets, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
|
||||
|
||||
@triton.jit
|
||||
def _transposed_overlap_add_kernel(
|
||||
z_ptr,
|
||||
bias_ptr,
|
||||
out_ptr,
|
||||
n_out_rows,
|
||||
t_len,
|
||||
c_len,
|
||||
STRIDE: tl.constexpr, # noqa: N803
|
||||
K: tl.constexpr, # noqa: N803
|
||||
BLOCK_R: tl.constexpr, # noqa: N803
|
||||
BLOCK_C: tl.constexpr, # noqa: N803
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R) # over B * T * STRIDE output steps
|
||||
chans = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
|
||||
row_mask = rows < n_out_rows
|
||||
chan_mask = chans < c_len
|
||||
mask = row_mask[:, None] & chan_mask[None, :]
|
||||
tap = rows % STRIDE
|
||||
src = rows // STRIDE # input row b * T + t
|
||||
acc = tl.load(
|
||||
z_ptr + (src[:, None].to(tl.int64) * K + tap[:, None]) * c_len + chans[None, :], mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
if K > STRIDE:
|
||||
prev_valid = row_mask & (src % t_len > 0)
|
||||
acc += tl.load(
|
||||
z_ptr + ((src[:, None].to(tl.int64) - 1) * K + tap[:, None] + STRIDE) * c_len + chans[None, :],
|
||||
mask=prev_valid[:, None] & chan_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += tl.load(bias_ptr + chans, mask=chan_mask, other=0.0).to(tl.float32)[None, :]
|
||||
tl.store(
|
||||
out_ptr + rows[:, None].to(tl.int64) * c_len + chans[None, :], acc.to(out_ptr.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def _blocks(c_len: int) -> tuple[int, int]:
|
||||
block_c = min(128, triton.next_power_of_2(c_len))
|
||||
return max(1, 4096 // block_c), block_c
|
||||
|
||||
|
||||
def snake_im2col(
|
||||
x: torch.Tensor,
|
||||
kernel_size: int,
|
||||
dilation: int = 1,
|
||||
bias: torch.Tensor | None = None,
|
||||
snake: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""``[B, T, C]`` -> ``[B * T, K * C]`` causal im2col with optional bias + SnakeBeta."""
|
||||
if not HAS_TRITON or not x.is_cuda:
|
||||
return snake_im2col_reference(x, kernel_size, dilation, bias, snake)
|
||||
x = x.contiguous()
|
||||
bsz, t, c = x.shape
|
||||
n_rows = bsz * t
|
||||
out = torch.empty(n_rows, kernel_size * c, dtype=x.dtype, device=x.device)
|
||||
block_r, block_c = _blocks(c)
|
||||
exp_alpha, inv_beta = snake if snake is not None else (x, x)
|
||||
_snake_im2col_kernel[(triton.cdiv(n_rows, block_r), triton.cdiv(c, block_c))](
|
||||
x,
|
||||
bias if bias is not None else x,
|
||||
exp_alpha,
|
||||
inv_beta,
|
||||
out,
|
||||
n_rows,
|
||||
t,
|
||||
c,
|
||||
dilation,
|
||||
K=kernel_size,
|
||||
HAS_BIAS=bias is not None,
|
||||
HAS_SNAKE=snake is not None,
|
||||
BLOCK_R=block_r,
|
||||
BLOCK_C=block_c,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def causal_conv(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
kernel_size: int,
|
||||
dilation: int = 1,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Causal dilated conv ``[B, T, C]`` -> ``[B, T, C_out]`` with a :func:`conv_gemm_weight` weight.
|
||||
|
||||
``residual`` (``[B, T, C_out]``) is added in the epilogue.
|
||||
"""
|
||||
if not HAS_TRITON or not x.is_cuda:
|
||||
return causal_conv_reference(x, weight, bias, kernel_size, dilation, residual)
|
||||
x = x.contiguous()
|
||||
bsz, t, c = x.shape
|
||||
n_rows, n_cols = bsz * t, weight.shape[1]
|
||||
if weight.shape[0] != kernel_size * c:
|
||||
raise ValueError(f"conv weight has {weight.shape[0]} rows, expected {kernel_size} taps x {c} channels")
|
||||
if max(n_rows * max(c, n_cols), weight.numel()) >= 2**31:
|
||||
raise ValueError("causal_conv offsets are 32-bit; split the batch")
|
||||
out = torch.empty(bsz, t, n_cols, dtype=x.dtype, device=x.device)
|
||||
if residual is not None:
|
||||
residual = residual.contiguous()
|
||||
|
||||
def grid(meta):
|
||||
return (triton.cdiv(n_rows, meta["BLOCK_M"]) * triton.cdiv(n_cols, meta["BLOCK_N"]),)
|
||||
|
||||
_causal_conv_gemm_kernel[grid](
|
||||
x,
|
||||
weight,
|
||||
bias if bias is not None else x,
|
||||
residual if residual is not None else x,
|
||||
out,
|
||||
n_rows,
|
||||
n_cols,
|
||||
c,
|
||||
t,
|
||||
dilation,
|
||||
triton.next_power_of_2(triton.cdiv(n_rows, 1024)),
|
||||
K=kernel_size,
|
||||
HAS_BIAS=bias is not None,
|
||||
HAS_RES=residual is not None,
|
||||
PRECISION="ieee" if x.dtype == torch.float32 else "tf32",
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def transposed_overlap_add(z: torch.Tensor, stride: int, bias: torch.Tensor) -> torch.Tensor:
|
||||
"""``[B, T, K, C_out]`` GEMM output of a kernel-``K`` transposed conv -> ``[B, T * stride, C_out]``."""
|
||||
bsz, t, k, c_out = z.shape
|
||||
if k not in (stride, 2 * stride):
|
||||
raise ValueError(f"transposed_overlap_add supports kernel stride or 2*stride, got kernel={k} stride={stride}")
|
||||
if not HAS_TRITON or not z.is_cuda:
|
||||
return transposed_overlap_add_reference(z, stride, bias)
|
||||
z = z.contiguous()
|
||||
n_out_rows = bsz * t * stride
|
||||
out = torch.empty(bsz, t * stride, c_out, dtype=z.dtype, device=z.device)
|
||||
block_r, block_c = _blocks(c_out)
|
||||
_transposed_overlap_add_kernel[(triton.cdiv(n_out_rows, block_r), triton.cdiv(c_out, block_c))](
|
||||
z, bias, out, n_out_rows, t, c_out, STRIDE=stride, K=k, BLOCK_R=block_r, BLOCK_C=block_c
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Time-major upsample + decoder conv stack for the Qwen3-TTS 12 Hz codec.
|
||||
|
||||
Same math as ``Qwen3TTSTokenizerV2Decoder.upsample``/``.decoder`` on ``[B, T, C]``
|
||||
activations: dense causal convs are implicit GEMMs (tensor cores, no im2col
|
||||
buffer, residual adds in the epilogue), SnakeBeta one pass per activation,
|
||||
transposed convs a GEMM plus an overlap-add. See ``time_major_conv.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.time_major_conv import (
|
||||
causal_conv,
|
||||
conv_gemm_weight,
|
||||
snake_im2col,
|
||||
transposed_gemm_weight,
|
||||
transposed_overlap_add,
|
||||
)
|
||||
|
||||
|
||||
def _snake(module) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if module._exp_alpha is None:
|
||||
module.precompute_exp_cache()
|
||||
return module._exp_alpha, module._inv_beta
|
||||
|
||||
|
||||
class _Conv:
|
||||
def __init__(self, causal_conv) -> None:
|
||||
conv = causal_conv.conv
|
||||
if conv.stride[0] != 1 or conv.groups != 1:
|
||||
raise ValueError("time-major decoder convs must be dense with stride 1")
|
||||
self.weight = conv_gemm_weight(conv.weight)
|
||||
self.bias = conv.bias.detach()
|
||||
self.kernel_size = conv.kernel_size[0]
|
||||
self.dilation = conv.dilation[0]
|
||||
|
||||
def __call__(self, x: torch.Tensor, residual: torch.Tensor | None = None) -> torch.Tensor:
|
||||
return causal_conv(x, self.weight, self.bias, self.kernel_size, self.dilation, residual)
|
||||
|
||||
|
||||
class _Transposed:
|
||||
def __init__(self, causal_trans_conv) -> None:
|
||||
conv = causal_trans_conv.conv
|
||||
self.weight = transposed_gemm_weight(conv.weight)
|
||||
self.bias = conv.bias.detach()
|
||||
self.kernel_size = conv.kernel_size[0]
|
||||
self.stride = conv.stride[0]
|
||||
self.out_channels = conv.out_channels
|
||||
|
||||
def __call__(self, x2d: torch.Tensor, bsz: int, t: int) -> torch.Tensor:
|
||||
z = torch.mm(x2d, self.weight).view(bsz, t, self.kernel_size, self.out_channels)
|
||||
return transposed_overlap_add(z, self.stride, self.bias)
|
||||
|
||||
|
||||
class TimeMajorConvStack:
|
||||
"""Callable replacing the NCT conv loops; holds GEMM-layout copies of the weights."""
|
||||
|
||||
def __init__(self, decoder: nn.Module) -> None:
|
||||
self.upsample = [(_Transposed(trans), block) for trans, block in decoder.upsample]
|
||||
self.conv_in = _Conv(decoder.decoder[0])
|
||||
self.blocks = []
|
||||
for block in decoder.decoder[1:-2]:
|
||||
snake, trans, *units = block.block
|
||||
residuals = [(_snake(u.act1), _Conv(u.conv1), _snake(u.act2), _Conv(u.conv2)) for u in units]
|
||||
self.blocks.append((_snake(snake), _Transposed(trans), residuals))
|
||||
self.snake_out = _snake(decoder.decoder[-2])
|
||||
self.conv_out = _Conv(decoder.decoder[-1])
|
||||
|
||||
@staticmethod
|
||||
def _convnext(x: torch.Tensor, block) -> torch.Tensor:
|
||||
dw = block.dwconv.conv
|
||||
k, t = dw.kernel_size[0], x.shape[1]
|
||||
xp = torch.nn.functional.pad(x, (0, 0, k - 1, 0))
|
||||
weight = dw.weight[:, 0, :].float()
|
||||
h = dw.bias.float() + sum(xp[:, j : j + t].float() * weight[:, j] for j in range(k))
|
||||
h = block.pwconv2(block.act(block.pwconv1(block.norm(h.to(x.dtype)))))
|
||||
return x + block.gamma * h
|
||||
|
||||
def __call__(self, hidden: torch.Tensor) -> torch.Tensor:
|
||||
"""``hidden`` ``[B, T, latent]`` -> unclamped waveform ``[B, 1, T * upsample]``."""
|
||||
bsz = hidden.shape[0]
|
||||
|
||||
def snake(x, params):
|
||||
return snake_im2col(x, 1, snake=params).view(x.shape)
|
||||
|
||||
for trans, block in self.upsample:
|
||||
t = hidden.shape[1]
|
||||
hidden = self._convnext(trans(hidden.reshape(bsz * t, -1), bsz, t), block)
|
||||
hidden = self.conv_in(hidden)
|
||||
for block_snake, trans, residuals in self.blocks:
|
||||
t = hidden.shape[1]
|
||||
hidden = trans(snake_im2col(hidden, 1, snake=block_snake), bsz, t)
|
||||
for act1, conv1, act2, conv2 in residuals:
|
||||
hidden = conv2(snake(conv1(snake(hidden, act1)), act2), residual=hidden)
|
||||
wav = self.conv_out(snake(hidden, self.snake_out))
|
||||
return wav.view(bsz, 1, -1)
|
||||
Reference in new issue
Block a user