[TTS][Perf] Optimize Qwen3-TTS high-concurrency serving (#3662)

Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-05-17 16:58:59 -04:00
1 parent 0a395f9de1
commit c99df1ebd9
14 files changed
+2703 -147

No files matched your search

@@ -467,10 +467,26 @@ class CodePredictorWrapper(nn.Module):
self._model_dtype: torch.dtype | None = None
self._compiled_model_fwd = None
self._bucket_sizes: list[int] = []
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
self._bucket_pos_ids: dict[int | tuple[int, int], torch.Tensor] = {}
self._lm_heads_list: list[nn.Module] | None = None
self._codec_embeds_list: list[nn.Module] | None = None
self._device_graphs: dict[int, tuple] = {} # (graph, static_output) per bucket
self._device_graphs: dict[int | tuple[int, int], tuple] = {} # (graph, static_output) per bucket
prefix_graph_cfg = self._stage_connector_extra_config(vllm_config)
prefix_graphs_requested = self._parse_bool_config(prefix_graph_cfg.get("code_predictor_prefix_graphs"))
is_npu = current_omni_platform.is_npu()
self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu
if prefix_graphs_requested and not self._prefix_graphs_enabled:
logger.info_once(
"code_predictor: prefix CUDA graphs requested but disabled because use_cuda_graphs=%s is_npu=%s",
wrapper_config.use_cuda_graphs,
is_npu,
)
self._prefix_graph_buckets = self._parse_positive_int_set(
prefix_graph_cfg.get("code_predictor_prefix_graph_buckets")
)
self._prefix_graph_seq_lens = self._parse_positive_int_set(
prefix_graph_cfg.get("code_predictor_prefix_graph_seq_lens")
)
def get_input_embeddings(self) -> nn.ModuleList:
return self.model.get_input_embeddings()
@@ -546,6 +562,56 @@ class CodePredictorWrapper(nn.Module):
return bucket
return bsz
@staticmethod
def _stage_connector_extra_config(vllm_config: VllmConfig) -> dict:
model_cfg = getattr(vllm_config, "model_config", None)
connector_cfg = getattr(model_cfg, "stage_connector_config", None)
if isinstance(connector_cfg, dict):
extra_cfg = connector_cfg.get("extra", connector_cfg)
else:
extra_cfg = getattr(connector_cfg, "extra", None)
return extra_cfg if isinstance(extra_cfg, dict) else {}
@staticmethod
def _parse_bool_config(value: object) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in ("1", "true", "yes", "on")
if isinstance(value, int):
return bool(value)
return False
@staticmethod
def _parse_positive_int_set(value: object) -> set[int]:
if value is None:
return set()
if isinstance(value, str):
raw_values = [item.strip() for item in value.replace(";", ",").split(",") if item.strip()]
elif isinstance(value, int):
raw_values = [value]
else:
try:
raw_values = list(value)
except TypeError as exc:
raise ValueError(f"Invalid positive int config value {value!r}") from exc
values: set[int] = set()
for item in raw_values:
try:
parsed = int(item)
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid positive int config value {item!r}") from exc
if parsed > 0:
values.add(parsed)
return values
def _prefix_seq_lens(self, max_seq: int) -> list[int]:
all_seq_lens = list(range(2, max_seq))
if not self._prefix_graph_seq_lens:
return all_seq_lens
allowed = set(all_seq_lens)
return sorted(seq_len for seq_len in self._prefix_graph_seq_lens if seq_len in allowed)
def _warmup_buckets(self) -> None:
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
@@ -562,12 +628,44 @@ class CodePredictorWrapper(nn.Module):
self._ensure_buffers(device, self._model_dtype, max(self._bucket_sizes))
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
self._bucket_pos_ids[bsz] = pos_ids
for _ in range(3):
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
if self._prefix_graphs_enabled:
prefix_seq_lens = self._prefix_seq_lens(max_seq)
needs_full_graph = set(prefix_seq_lens) != set(range(2, max_seq))
for bsz in self._bucket_sizes:
capture_prefixes = not self._prefix_graph_buckets or bsz in self._prefix_graph_buckets
if not capture_prefixes or needs_full_graph:
pos_ids = (
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
)
self._bucket_pos_ids[bsz] = pos_ids
for _ in range(3):
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
if capture_prefixes:
for seq_len in prefix_seq_lens:
pos_ids = (
torch.arange(seq_len, device=device, dtype=torch.long)
.unsqueeze(0)
.expand(bsz, -1)
.contiguous()
)
self._bucket_pos_ids[(bsz, seq_len)] = pos_ids
for _ in range(2):
self._compiled_model_fwd(proj_buf[:bsz, :seq_len, :], pos_ids)
logger.info(
"code_predictor: prefix warmup done for buckets %s prefix_buckets=%s seq_lens=%s",
self._bucket_sizes,
sorted(self._prefix_graph_buckets) if self._prefix_graph_buckets else "all",
prefix_seq_lens,
)
else:
for bsz in self._bucket_sizes:
pos_ids = (
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
)
self._bucket_pos_ids[bsz] = pos_ids
for _ in range(3):
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
def _capture_cuda_graphs(self) -> None:
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
@@ -577,17 +675,50 @@ class CodePredictorWrapper(nn.Module):
max_seq = self._num_groups + 1
proj_buf = self._proj_buf
for bsz in self._bucket_sizes:
static_input = proj_buf[:bsz, :max_seq, :]
pos_ids = self._bucket_pos_ids[bsz]
if self._prefix_graphs_enabled:
prefix_seq_lens = self._prefix_seq_lens(max_seq)
needs_full_graph = set(prefix_seq_lens) != set(range(2, max_seq))
for bsz in self._bucket_sizes:
capture_prefixes = not self._prefix_graph_buckets or bsz in self._prefix_graph_buckets
if not capture_prefixes or needs_full_graph:
static_input = proj_buf[:bsz, :max_seq, :]
pos_ids = self._bucket_pos_ids[bsz]
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
self._device_graphs[bsz] = (g, static_output)
self._device_graphs[bsz] = (g, static_output)
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
if capture_prefixes:
for seq_len in prefix_seq_lens:
static_input = proj_buf[:bsz, :seq_len, :]
pos_ids = self._bucket_pos_ids[(bsz, seq_len)]
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
self._device_graphs[(bsz, seq_len)] = (g, static_output)
logger.info(
"code_predictor: captured prefix CUDA graphs for buckets %s prefix_buckets=%s seq_lens=%s",
self._bucket_sizes,
sorted(self._prefix_graph_buckets) if self._prefix_graph_buckets else "all",
prefix_seq_lens,
)
else:
for bsz in self._bucket_sizes:
static_input = proj_buf[:bsz, :max_seq, :]
pos_ids = self._bucket_pos_ids[bsz]
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g, pool=pool):
static_output = self._compiled_model_fwd(static_input, pos_ids)
self._device_graphs[bsz] = (g, static_output)
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
def _capture_npu_graphs(self) -> None:
"""Capture an NPU graph per bucket using torch_npu's NPUGraph."""
@@ -650,16 +781,6 @@ class CodePredictorWrapper(nn.Module):
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
# Get pre-computed pos_ids for this bucket
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
if full_pos_ids is None:
full_pos_ids = (
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
)
# Use captured device graph if available, otherwise call compiled fn.
device_graph_entry = self._device_graphs.get(padded_bsz)
# Prepare sampling parameters
stored_mode = self._wrapper_config.sampling_mode == "stored"
if stored_mode:
@@ -683,12 +804,31 @@ class CodePredictorWrapper(nn.Module):
# Autoregressive loop: predict layers 1..G-1
for step in range(1, num_groups):
graph_key: int | tuple[int, int] = padded_bsz
seq_len = max_seq
if self._prefix_graphs_enabled:
prefix_key = (padded_bsz, step + 1)
if prefix_key in self._device_graphs:
graph_key = prefix_key
seq_len = step + 1
pos_ids = self._bucket_pos_ids.get(graph_key)
if pos_ids is None:
pos_ids = (
torch.arange(seq_len, device=device, dtype=torch.long)
.unsqueeze(0)
.expand(padded_bsz, -1)
.contiguous()
)
# Use captured device graph if available, otherwise call compiled fn.
device_graph_entry = self._device_graphs.get(graph_key)
# Run transformer (device graph replay or compiled forward)
if device_graph_entry is not None:
device_graph_entry[0].replay()
hidden_out = device_graph_entry[1]
else:
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
@@ -7,6 +7,11 @@ This module provides CUDA Graph acceleration for the speech tokenizer decoder,
reducing kernel launch overhead during inference.
"""
import os
import time
from collections import Counter
from collections.abc import Callable, Sequence
import torch
from torch.cuda import CUDAGraph
from vllm.logger import init_logger
@@ -15,6 +20,108 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
def _normalize_decode_lengths(lengths: Sequence[int], batch_size: int, max_len: int) -> tuple[int, ...]:
if len(lengths) != batch_size:
raise ValueError(f"Expected {batch_size} decode lengths, got {len(lengths)}")
normalized: list[int] = []
for length in lengths:
length_int = int(length)
if length_int < 0 or length_int > max_len:
raise ValueError(f"Invalid decode length {length_int}; expected 0 <= length <= {max_len}")
normalized.append(length_int)
return tuple(normalized)
def _batched_chunked_decode(
codes: torch.Tensor,
lengths: Sequence[int],
*,
decode_fn: Callable[[torch.Tensor], torch.Tensor],
total_upsample: int,
chunk_size: int = 300,
left_context_size: int = 25,
max_batch_size: int = 0,
) -> torch.Tensor:
"""Decode a padded batch by grouping same-round chunks across requests."""
if codes.dim() < 3:
raise ValueError(f"Expected codes with shape [B, Q, F], got {tuple(codes.shape)}")
if chunk_size <= 0:
raise ValueError(f"chunk_size must be positive, got {chunk_size}")
if left_context_size < 0:
raise ValueError(f"left_context_size must be non-negative, got {left_context_size}")
if max_batch_size < 0:
raise ValueError(f"max_batch_size must be non-negative, got {max_batch_size}")
batch_size = int(codes.shape[0])
max_input_len = int(codes.shape[-1])
length_values = _normalize_decode_lengths(lengths, batch_size, max_input_len)
max_decode_len = max(length_values, default=0)
if max_decode_len == 0:
return torch.empty((batch_size, 1, 0), dtype=torch.float32, device=codes.device)
total_upsample = int(total_upsample)
wav_out: torch.Tensor | None = None
num_rounds = (max_decode_len + chunk_size - 1) // chunk_size
for round_index in range(num_rounds):
start_index = round_index * chunk_size
grouped_jobs: dict[int, list[tuple[int, int, int, int, int, int]]] = {}
for req_index, total_len in enumerate(length_values):
if start_index >= total_len:
continue
end_index = min(start_index + chunk_size, total_len)
context_size = left_context_size if start_index - left_context_size > 0 else start_index
input_start = start_index - context_size
input_end = end_index
chunk_len = input_end - input_start
grouped_jobs.setdefault(chunk_len, []).append(
(req_index, input_start, input_end, start_index, end_index, context_size)
)
for jobs in grouped_jobs.values():
job_batches = (
[jobs]
if max_batch_size <= 0 or len(jobs) <= max_batch_size
else [jobs[start : start + max_batch_size] for start in range(0, len(jobs), max_batch_size)]
)
for job_batch in job_batches:
chunk_rows = [
codes[req_index, :, input_start:input_end]
for req_index, input_start, input_end, _, _, _ in job_batch
]
codes_chunk = torch.stack(
chunk_rows,
dim=0,
)
wav_chunk = decode_fn(codes_chunk)
if wav_chunk.shape[0] != len(job_batch):
raise ValueError(
f"Decoder returned batch size {wav_chunk.shape[0]} for input batch size {len(job_batch)}"
)
if wav_out is None:
wav_out = torch.empty(
(batch_size, *wav_chunk.shape[1:-1], max_decode_len * total_upsample),
dtype=wav_chunk.dtype,
device=wav_chunk.device,
)
for row, (req_index, _, _, chunk_start, chunk_end, context_size) in enumerate(job_batch):
src_start = context_size * total_upsample
dst_start = chunk_start * total_upsample
dst_end = chunk_end * total_upsample
src_end = src_start + (dst_end - dst_start)
if src_end > wav_chunk.shape[-1]:
raise ValueError(
f"Decoder returned too-short chunk output: need {src_end}, got {wav_chunk.shape[-1]}"
)
wav_out[req_index, ..., dst_start:dst_end].copy_(wav_chunk[row, ..., src_start:src_end])
if wav_out is None:
return torch.empty((batch_size, 1, 0), dtype=torch.float32, device=codes.device)
return wav_out
class CUDAGraphDecoderWrapper:
"""
CUDA Graph wrapper for Qwen3TTSTokenizerV2Decoder.
@@ -34,21 +141,61 @@ class CUDAGraphDecoderWrapper:
self,
decoder: torch.nn.Module,
capture_sizes: list[int] | None = None,
capture_batch_sizes: list[int] | None = None,
extra_capture_shapes: list[tuple[int, int]] | None = None,
compile_shapes: list[tuple[int, int]] | None = None,
num_quantizers: int = 8,
enabled: bool = True,
):
self.decoder = decoder
self._explicit_sizes = capture_sizes is not None
self.capture_sizes = sorted(capture_sizes) if capture_sizes else []
self.capture_batch_sizes = sorted(set(capture_batch_sizes or [1]))
self.extra_capture_shapes = sorted(
{
(int(batch_size), int(size))
for batch_size, size in extra_capture_shapes or []
if int(batch_size) > 0 and int(size) > 0
}
)
self.compile_shapes = sorted(
{
(int(batch_size), int(size))
for batch_size, size in compile_shapes or []
if int(batch_size) > 0 and int(size) > 0
}
)
self._bucket_sizes = self.capture_sizes
self.num_quantizers = num_quantizers
self.enabled = enabled
self.graphs: dict[int, CUDAGraph] = {}
self.static_inputs: dict[int, torch.Tensor] = {}
self.static_outputs: dict[int, torch.Tensor] = {}
self.graphs: dict[tuple[int, int], CUDAGraph] = {}
self.static_inputs: dict[tuple[int, int], torch.Tensor] = {}
self.static_outputs: dict[tuple[int, int], torch.Tensor] = {}
self._compiled_decoder: Callable[[torch.Tensor], torch.Tensor] | None = None
self._compiled_graphs: dict[tuple[int, int], CUDAGraph] = {}
self._compiled_static_inputs: dict[tuple[int, int], torch.Tensor] = {}
self._compiled_static_outputs: dict[tuple[int, int], torch.Tensor] = {}
self._compiled_shapes: set[tuple[int, int]] = set()
self._warmed_up = False
self._device = None
self._stats_enabled = os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_CUDAGRAPH_STATS", "").lower() in (
"1",
"true",
"yes",
"on",
)
self._stats_log_every = int(os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_CUDAGRAPH_STATS_LOG_EVERY", "0") or 0)
self._stats_total = 0
self._stats_hits = 0
self._stats_compiled_hits = 0
self._stats_fallbacks = 0
self._stats_stream_capture_fallbacks = 0
self._stats_requests: Counter[tuple[int, int]] = Counter()
self._stats_hit_shapes: Counter[tuple[int, int, int]] = Counter()
self._stats_compiled_shapes: Counter[tuple[int, int]] = Counter()
self._stats_fallback_shapes: Counter[tuple[int, int, int]] = Counter()
@staticmethod
def compute_capture_sizes(
@@ -78,11 +225,68 @@ class CUDAGraphDecoderWrapper:
return sorted(sizes)
def _get_padded_size(self, actual_size: int) -> int | None:
for size in self.capture_sizes:
for size in self._bucket_sizes:
if actual_size <= size:
return size
return None
def _get_capture_shapes(self) -> list[tuple[int, int]]:
shapes = {(batch_size, size) for batch_size in self.capture_batch_sizes for size in self.capture_sizes}
shapes.update(self.extra_capture_shapes)
return sorted(shapes)
def _record_decode_stats(
self,
*,
hit: bool,
batch_size: int,
actual_size: int,
padded_size: int | None,
stream_capture: bool = False,
compiled: bool = False,
) -> None:
if not self._stats_enabled:
return
padded_key = int(padded_size) if padded_size is not None else -1
self._stats_total += 1
self._stats_requests[(batch_size, actual_size)] += 1
if hit:
self._stats_hits += 1
if compiled:
self._stats_compiled_hits += 1
self._stats_compiled_shapes[(batch_size, actual_size)] += 1
else:
self._stats_hit_shapes[(batch_size, actual_size, padded_key)] += 1
else:
self._stats_fallbacks += 1
self._stats_fallback_shapes[(batch_size, actual_size, padded_key)] += 1
if stream_capture:
self._stats_stream_capture_fallbacks += 1
if self._stats_log_every > 0 and self._stats_total % self._stats_log_every == 0:
self.log_decode_stats()
def log_decode_stats(self) -> None:
if not self._stats_enabled or self._stats_total == 0:
return
hit_rate = 100.0 * self._stats_hits / self._stats_total
logger.info(
"Code2Wav CUDA Graph stats: total=%d hits=%d fallbacks=%d "
"compiled_hits=%d stream_capture_fallbacks=%d hit_rate=%.2f%% "
"top_requests=%s top_compiled=%s top_hits=%s top_fallbacks=%s",
self._stats_total,
self._stats_hits,
self._stats_fallbacks,
self._stats_compiled_hits,
self._stats_stream_capture_fallbacks,
hit_rate,
self._stats_requests.most_common(12),
self._stats_compiled_shapes.most_common(12),
self._stats_hit_shapes.most_common(12),
self._stats_fallback_shapes.most_common(12),
)
def warmup(
self,
device: torch.device,
@@ -106,28 +310,97 @@ class CUDAGraphDecoderWrapper:
decode_left_context=decode_left_context,
)
logger.info("Starting CUDA Graph warmup for %d sizes: %s", len(self.capture_sizes), self.capture_sizes)
self.capture_batch_sizes = [bs for bs in self.capture_batch_sizes if bs > 0]
if not self.capture_batch_sizes:
self.capture_batch_sizes = [1]
self._bucket_sizes = sorted(set(self.capture_sizes) | {size for _, size in self.extra_capture_shapes})
capture_shapes = self._get_capture_shapes()
logger.info(
"Starting CUDA Graph warmup for %d shapes: batch_sizes=%s seq_lens=%s extra_shapes=%s",
len(capture_shapes),
self.capture_batch_sizes,
self.capture_sizes,
self.extra_capture_shapes,
)
warmup_start_s = time.perf_counter()
mem_before = self._get_cuda_memory_stats(device)
# Warmup runs to ensure CUDA memory is allocated
for size in self.capture_sizes:
dummy = torch.zeros(1, self.num_quantizers, size, dtype=dtype, device=device)
for batch_size, size in capture_shapes:
dummy = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
with torch.no_grad():
_ = self.decoder(dummy)
torch.accelerator.synchronize(device)
for size in self.capture_sizes:
for batch_size, size in capture_shapes:
try:
self._capture(size, device, dtype)
logger.info(" Captured CUDA Graph for size=%d", size)
self._capture(batch_size, size, device, dtype)
logger.info(" Captured CUDA Graph for batch=%d size=%d", batch_size, size)
except Exception:
logger.warning(" Failed to capture graph for size=%d", size, exc_info=True)
logger.warning(" Failed to capture graph for batch=%d size=%d", batch_size, size, exc_info=True)
if self.compile_shapes:
self._warmup_compile_shapes(device, dtype)
self._warmed_up = True
logger.info("CUDA Graph warmup complete: %d/%d captured", len(self.graphs), len(self.capture_sizes))
warmup_ms = (time.perf_counter() - warmup_start_s) * 1000.0
mem_after = self._get_cuda_memory_stats(device)
logger.info(
"CUDA Graph warmup complete: %d/%d captured in %.1f ms%s",
len(self.graphs),
len(capture_shapes),
warmup_ms,
self._format_cuda_memory_delta(mem_before, mem_after),
)
def _capture(self, size: int, device: torch.device, dtype: torch.dtype):
static_input = torch.zeros(1, self.num_quantizers, size, dtype=dtype, device=device)
def _warmup_compile_shapes(self, device: torch.device, dtype: torch.dtype) -> None:
logger.info("Starting torch.compile + CUDA Graph warmup for decoder shapes: %s", self.compile_shapes)
compile_start_s = time.perf_counter()
try:
self._compiled_decoder = torch.compile(
self.decoder.forward,
mode="default",
fullgraph=False,
dynamic=False,
)
except Exception:
logger.warning("Failed to create torch.compile decoder wrapper", exc_info=True)
self._compiled_decoder = None
return
assert self._compiled_decoder is not None
for batch_size, size in self.compile_shapes:
shape_start_s = time.perf_counter()
try:
self._capture_compiled(batch_size, size, device, dtype)
self._compiled_shapes.add((batch_size, size))
logger.info(
" torch.compile + CUDA Graph ready for batch=%d size=%d in %.1f ms",
batch_size,
size,
(time.perf_counter() - shape_start_s) * 1000.0,
)
except Exception:
logger.warning(
" Failed to capture torch.compile CUDA Graph for batch=%d size=%d; "
"this shape will use CUDA Graph/eager fallback",
batch_size,
size,
exc_info=True,
)
logger.info(
"torch.compile + CUDA Graph warmup complete: %d/%d shapes ready in %.1f ms",
len(self._compiled_shapes),
len(self.compile_shapes),
(time.perf_counter() - compile_start_s) * 1000.0,
)
def _capture(self, batch_size: int, size: int, device: torch.device, dtype: torch.dtype):
key = (batch_size, size)
static_input = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
with torch.no_grad():
_ = self.decoder(static_input)
torch.accelerator.synchronize(device)
@@ -137,12 +410,64 @@ class CUDAGraphDecoderWrapper:
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_output = self.decoder(static_input)
self.graphs[size] = graph
self.static_inputs[size] = static_input
self.static_outputs[size] = static_output
self.graphs[key] = graph
self.static_inputs[key] = static_input
self.static_outputs[key] = static_output
def decode(self, codes: torch.Tensor) -> torch.Tensor:
if not self.enabled or not self._warmed_up or codes.shape[0] != 1:
def _capture_compiled(self, batch_size: int, size: int, device: torch.device, dtype: torch.dtype):
if self._compiled_decoder is None:
raise RuntimeError("Compiled decoder is not initialized")
key = (batch_size, size)
static_input = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
with torch.inference_mode():
for _ in range(5):
_ = self._compiled_decoder(static_input)
torch.accelerator.synchronize(device)
graph = CUDAGraph()
with torch.inference_mode():
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_output = self._compiled_decoder(static_input)
self._compiled_graphs[key] = graph
self._compiled_static_inputs[key] = static_input
self._compiled_static_outputs[key] = static_output
@staticmethod
def _get_cuda_memory_stats(device: torch.device) -> tuple[int, int, int] | None:
if device.type != "cuda":
return None
try:
return (
int(torch.cuda.memory_allocated(device)),
int(torch.cuda.memory_reserved(device)),
int(torch.cuda.max_memory_reserved(device)),
)
except Exception:
return None
@staticmethod
def _format_cuda_memory_delta(
before: tuple[int, int, int] | None,
after: tuple[int, int, int] | None,
) -> str:
if before is None or after is None:
return ""
def gib(value: int) -> float:
return value / 1024**3
alloc_before, reserved_before, max_reserved_before = before
alloc_after, reserved_after, max_reserved_after = after
return (
f" (cuda_mem allocated {gib(alloc_before):.2f}->{gib(alloc_after):.2f} GiB, "
f"reserved {gib(reserved_before):.2f}->{gib(reserved_after):.2f} GiB, "
f"max_reserved {gib(max_reserved_before):.2f}->{gib(max_reserved_after):.2f} GiB)"
)
def _decode(self, codes: torch.Tensor, *, clone_graph_output: bool) -> torch.Tensor:
if not self.enabled or not self._warmed_up:
return self.decoder(codes)
# Inner CUDA graph replay is illegal while an outer stream capture is
@@ -152,20 +477,76 @@ class CUDAGraphDecoderWrapper:
# outside the startup capture window, so normal inference still hits
# the graph fast path.
if torch.cuda.is_current_stream_capturing():
self._record_decode_stats(
hit=False,
batch_size=int(codes.shape[0]),
actual_size=int(codes.shape[-1]),
padded_size=None,
stream_capture=True,
)
return self.decoder(codes)
actual_size = codes.shape[-1]
batch_size = int(codes.shape[0])
actual_size = int(codes.shape[-1])
padded_size = self._get_padded_size(actual_size)
compile_key = (batch_size, actual_size)
if compile_key not in self._compiled_shapes and padded_size is not None:
compile_key = (batch_size, padded_size)
if compile_key in self._compiled_shapes:
compiled_size = compile_key[1]
self._record_decode_stats(
hit=True,
batch_size=batch_size,
actual_size=actual_size,
padded_size=compiled_size,
compiled=True,
)
static_input = self._compiled_static_inputs[compile_key]
if actual_size == compiled_size:
static_input.copy_(codes)
else:
static_input.zero_()
static_input[:, :, :actual_size] = codes
self._compiled_graphs[compile_key].replay()
actual_out_len = actual_size * self.decoder.total_upsample
output = self._compiled_static_outputs[compile_key][..., :actual_out_len]
if clone_graph_output:
return output.clone()
return output
if padded_size is None or padded_size not in self.graphs:
graph_key = (batch_size, padded_size) if padded_size is not None else None
if graph_key is None or graph_key not in self.graphs:
self._record_decode_stats(
hit=False,
batch_size=batch_size,
actual_size=actual_size,
padded_size=padded_size,
)
return self.decoder(codes)
self.static_inputs[padded_size].zero_()
self.static_inputs[padded_size][:, :, :actual_size] = codes
self.graphs[padded_size].replay()
self._record_decode_stats(
hit=True,
batch_size=batch_size,
actual_size=actual_size,
padded_size=padded_size,
)
static_input = self.static_inputs[graph_key]
if actual_size == padded_size:
static_input.copy_(codes)
else:
static_input.zero_()
static_input[:, :, :actual_size] = codes
self.graphs[graph_key].replay()
actual_out_len = actual_size * self.decoder.total_upsample
return self.static_outputs[padded_size][..., :actual_out_len].clone()
output = self.static_outputs[graph_key][..., :actual_out_len]
if clone_graph_output:
return output.clone()
return output
def decode(self, codes: torch.Tensor) -> torch.Tensor:
return self._decode(codes, clone_graph_output=True)
def chunked_decode_with_cudagraph(
self,
@@ -173,19 +554,48 @@ class CUDAGraphDecoderWrapper:
chunk_size: int = 300,
left_context_size: int = 25,
) -> torch.Tensor:
wavs = []
start_index = 0
total_len = codes.shape[-1]
total_upsample = self.decoder.total_upsample
wav_out = None
while start_index < total_len:
end_index = min(start_index + chunk_size, total_len)
context_size = left_context_size if start_index - left_context_size > 0 else start_index
codes_chunk = codes[..., start_index - context_size : end_index]
wav_chunk = self.decode(codes_chunk)
wav_chunk = self._decode(codes_chunk, clone_graph_output=False)
wavs.append(wav_chunk[..., context_size * total_upsample :])
if wav_out is None:
wav_out = torch.empty(
(*wav_chunk.shape[:-1], total_len * total_upsample),
dtype=wav_chunk.dtype,
device=wav_chunk.device,
)
src_start = context_size * total_upsample
dst_start = start_index * total_upsample
dst_end = end_index * total_upsample
wav_out[..., dst_start:dst_end].copy_(wav_chunk[..., src_start:])
start_index = end_index
return torch.cat(wavs, dim=-1)
if wav_out is None:
return self.decoder(codes)
return wav_out
def batched_chunked_decode_with_cudagraph(
self,
codes: torch.Tensor,
lengths: Sequence[int],
chunk_size: int = 300,
left_context_size: int = 25,
max_batch_size: int = 0,
) -> torch.Tensor:
return _batched_chunked_decode(
codes,
lengths,
decode_fn=lambda codes_chunk: self._decode(codes_chunk, clone_graph_output=False),
total_upsample=self.decoder.total_upsample,
chunk_size=chunk_size,
left_context_size=left_context_size,
max_batch_size=max_batch_size,
)
@@ -1,5 +1,7 @@
from __future__ import annotations
import os
from collections import Counter
from collections.abc import Iterable
from typing import Any
@@ -43,7 +45,25 @@ class Qwen3TTSCode2Wav(nn.Module):
self._decode_chunk_frames = 300
self._decode_left_context_frames = 25
self._decode_batch_bucket_frames: list[int] = []
self._decode_batch_max_size = 0
self._decode_variable_chunk_batch_min_frames = self._decode_chunk_frames + self._decode_left_context_frames + 1
self._logged_codec_stats = False
self._logged_malformed_codec_lengths: set[tuple[int, int]] = set()
self._batch_stats_enabled = os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS", "").lower() in (
"1",
"true",
"yes",
"on",
)
self._batch_stats_log_every = int(os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS_LOG_EVERY", "0") or 0)
self._batch_stats_forwards = 0
self._batch_stats_groups = 0
self._batch_stats_requests = 0
self._batch_stats_padded_frames = 0
self._batch_stats_decoded_frames = 0
self._batch_stats_actual_frames: Counter[int] = Counter()
self._batch_stats_bucket_groups: Counter[tuple[int, int]] = Counter()
# Construct decoder from config so it is visible to vLLM's
# memory profiler at startup. Weights are loaded later in
@@ -91,6 +111,50 @@ class Qwen3TTSCode2Wav(nn.Module):
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
return [ids]
def _get_decode_batch_bucket_frames(self, actual_frames: int) -> int:
for bucket_frames in self._decode_batch_bucket_frames:
if actual_frames <= bucket_frames:
return bucket_frames
return actual_frames
def _record_decode_batch_stats(
self,
*,
group_size: int,
bucket_frames: int,
actual_frames: list[int],
) -> None:
if not self._batch_stats_enabled:
return
self._batch_stats_groups += 1
self._batch_stats_requests += group_size
self._batch_stats_decoded_frames += group_size * bucket_frames
self._batch_stats_padded_frames += sum(bucket_frames - frames for frames in actual_frames)
self._batch_stats_actual_frames.update(actual_frames)
self._batch_stats_bucket_groups[(group_size, bucket_frames)] += 1
def log_decode_batch_stats(self) -> None:
if not self._batch_stats_enabled or self._batch_stats_requests == 0:
return
avg_group_size = self._batch_stats_requests / max(1, self._batch_stats_groups)
pad_ratio = self._batch_stats_padded_frames / max(1, self._batch_stats_decoded_frames)
logger.info(
"Code2Wav batch stats: forwards=%d groups=%d requests=%d "
"avg_group_size=%.2f padded_frames=%d decoded_frames=%d pad_ratio=%.2f%% "
"top_actual_frames=%s top_bucket_groups=%s",
self._batch_stats_forwards,
self._batch_stats_groups,
self._batch_stats_requests,
avg_group_size,
self._batch_stats_padded_frames,
self._batch_stats_decoded_frames,
100.0 * pad_ratio,
self._batch_stats_actual_frames.most_common(12),
self._batch_stats_bucket_groups.most_common(12),
)
@torch.no_grad()
def forward(
self,
@@ -111,6 +175,7 @@ class Qwen3TTSCode2Wav(nn.Module):
Length management is done here instead of relying on HF's padding=-1
sentinel logic.
"""
self._batch_stats_forwards += 1
decoder = self.decoder
q = int(self._num_quantizers)
upsample = int(self._total_upsample)
@@ -147,11 +212,15 @@ class Qwen3TTSCode2Wav(nn.Module):
n = flat.numel()
if n == 0 or n % q != 0:
if n > 0:
logger.warning(
"Code2Wav input_ids length %d not divisible by num_quantizers %d; skipping malformed request.",
n,
q,
)
key = (int(n), q)
if key not in self._logged_malformed_codec_lengths:
self._logged_malformed_codec_lengths.add(key)
logger.warning(
"Code2Wav input_ids length %d not divisible by num_quantizers %d; "
"skipping malformed request and suppressing repeats for this length.",
n,
q,
)
parsed.append((0, 0))
continue
frames = n // q
@@ -187,22 +256,95 @@ class Qwen3TTSCode2Wav(nn.Module):
except Exception:
pass
# Decode directly via decoder.chunked_decode(), staying entirely on GPU.
# Each request decoded individually with CUDA graph replay at bs=1.
wav_tensors: list[torch.Tensor] = []
for codes_qf in valid_codes_qf:
codes_bqf = codes_qf.unsqueeze(0) # [1, Q, F]
try:
wav = decoder.chunked_decode(
codes_bqf,
chunk_size=self._decode_chunk_frames,
left_context_size=self._decode_left_context_frames,
) # [1, 1, wav_len]
except TypeError:
# Unit-test fakes and older decoder shims may not accept the
# explicit chunk kwargs; production Qwen3-TTS decoders do.
wav = decoder.chunked_decode(codes_bqf) # [1, 1, wav_len]
wav_tensors.append(wav.squeeze(0).squeeze(0)) # [wav_len]
wav_tensors: list[torch.Tensor | None] = [None] * len(valid_codes_qf)
def _decode_group_chunks(group_chunks: list[list[tuple[int, torch.Tensor]]]) -> None:
for group_chunk in group_chunks:
actual_frames = [int(codes_qf.shape[1]) for _, codes_qf in group_chunk]
target_frames = max(actual_frames)
is_equal_length_batch = all(frames == target_frames for frames in actual_frames)
use_variable_length_batch = (
len(group_chunk) > 1
and not is_equal_length_batch
and target_frames >= self._decode_variable_chunk_batch_min_frames
and hasattr(decoder, "batched_chunked_decode")
)
if len(group_chunk) == 1:
codes_bqf = group_chunk[0][1].unsqueeze(0)
elif is_equal_length_batch:
codes_bqf = torch.stack([codes_qf for _, codes_qf in group_chunk], dim=0)
else:
first = group_chunk[0][1]
codes_bqf = first.new_zeros((len(group_chunk), q, target_frames))
for row, (_, codes_qf) in enumerate(group_chunk):
codes_bqf[row, :, : codes_qf.shape[1]] = codes_qf
self._record_decode_batch_stats(
group_size=len(group_chunk),
bucket_frames=target_frames,
actual_frames=actual_frames,
)
try:
if use_variable_length_batch:
wav_batch = decoder.batched_chunked_decode(
codes_bqf,
actual_frames,
chunk_size=self._decode_chunk_frames,
left_context_size=self._decode_left_context_frames,
max_batch_size=self._decode_batch_max_size,
) # [B, 1, wav_len]
else:
wav_batch = decoder.chunked_decode(
codes_bqf,
chunk_size=self._decode_chunk_frames,
left_context_size=self._decode_left_context_frames,
) # [B, 1, wav_len]
except TypeError:
# Unit-test fakes and older decoder shims may not accept the
# explicit chunk kwargs; production Qwen3-TTS decoders do.
wav_batch = decoder.chunked_decode(codes_bqf) # [B, 1, wav_len]
if wav_batch.dim() == 3 and wav_batch.shape[1] == 1:
wav_rows = wav_batch[:, 0, :]
elif wav_batch.dim() == 2:
wav_rows = wav_batch
else:
raise ValueError(
"Code2Wav decoder returned unexpected shape "
f"{tuple(wav_batch.shape)} for batch size {len(group_chunk)}"
)
if wav_rows.shape[0] != len(group_chunk):
raise ValueError(
f"Code2Wav decoder returned batch size {wav_rows.shape[0]} "
f"for input batch size {len(group_chunk)}"
)
for row, (j, _) in enumerate(group_chunk):
wav_tensors[j] = wav_rows[row]
# Group by configured frame buckets instead of only exact lengths.
# For ordinary async streaming windows this is the real batching
# opportunity; decoder-internal variable chunk batching is gated to
# longer inputs where repeated full chunks can amortize its overhead.
grouped_codes: dict[int, list[tuple[int, torch.Tensor]]] = {}
for j, codes_qf in enumerate(valid_codes_qf):
frames = int(codes_qf.shape[1])
grouped_codes.setdefault(self._get_decode_batch_bucket_frames(frames), []).append((j, codes_qf))
for _bucket_frames, group in grouped_codes.items():
if self._decode_batch_max_size > 0 and len(group) > self._decode_batch_max_size:
# Keep each decoder call inside the configured CUDA graph batch
# envelope. Sorting by length lowers right-padding within each
# split while outputs are restored by original request index.
group = sorted(group, key=lambda item: int(item[1].shape[1]))
group_chunks = [
group[start : start + self._decode_batch_max_size]
for start in range(0, len(group), self._decode_batch_max_size)
]
else:
group_chunks = [group]
_decode_group_chunks(group_chunks)
if self._batch_stats_log_every > 0 and self._batch_stats_forwards % self._batch_stats_log_every == 0:
self.log_decode_batch_stats()
audios: list[torch.Tensor] = [empty] * num_req
srs = [sr_tensor] * num_req
@@ -210,6 +352,7 @@ class Qwen3TTSCode2Wav(nn.Module):
for j, idx in enumerate(valid_indices):
ctx_frames, actual_frames = parsed[idx]
wav = wav_tensors[j]
assert wav is not None
# Slice on exact codec-frame boundaries instead of proportionally.
start = max(0, ctx_frames * upsample)
end = max(start, actual_frames * upsample)
@@ -302,6 +445,80 @@ class Qwen3TTSCode2Wav(nn.Module):
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
def _get_bool_config(name: str, default: bool) -> bool:
value = extra_cfg.get(name, default)
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in ("1", "true", "yes", "on"):
return True
if lowered in ("0", "false", "no", "off"):
return False
if isinstance(value, int):
return bool(value)
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
def _get_int_list_config(name: str) -> list[int] | None:
value = extra_cfg.get(name)
if value is None:
return None
if isinstance(value, str):
raw_values = [item.strip() for item in value.split(",") if item.strip()]
elif isinstance(value, int):
raw_values = [value]
else:
try:
raw_values = list(value)
except TypeError as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
values: set[int] = set()
for item in raw_values:
try:
parsed = int(item)
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
if parsed > 0:
values.add(parsed)
return sorted(values)
def _get_int_pair_list_config(name: str) -> list[tuple[int, int]] | None:
value = extra_cfg.get(name)
if value is None:
return None
if isinstance(value, str):
raw_values = [item.strip() for item in value.split(",") if item.strip()]
else:
try:
raw_values = list(value)
except TypeError as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
pairs: set[tuple[int, int]] = set()
for item in raw_values:
if isinstance(item, str):
if ":" not in item:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
left, right = item.split(":", 1)
raw_pair = (left.strip(), right.strip())
else:
try:
raw_pair = tuple(item)
except TypeError as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
if len(raw_pair) != 2:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
try:
batch_size = int(raw_pair[0])
seq_len = int(raw_pair[1])
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
if batch_size > 0 and seq_len > 0:
pairs.add((batch_size, seq_len))
return sorted(pairs)
if isinstance(extra_cfg, dict):
codec_chunk_frames = int(extra_cfg.get("codec_chunk_frames") or 0)
codec_left_context_frames = int(extra_cfg.get("codec_left_context_frames") or 0)
@@ -318,6 +535,49 @@ class Qwen3TTSCode2Wav(nn.Module):
)
self._decode_chunk_frames = decode_chunk_frames
self._decode_left_context_frames = decode_left_context_frames
decode_cudagraph_capture_sizes = _get_int_list_config("decode_cudagraph_capture_sizes")
decode_cudagraph_batch_sizes = _get_int_list_config("decode_cudagraph_batch_sizes")
decode_cudagraph_extra_capture_shapes = _get_int_pair_list_config("decode_cudagraph_extra_capture_shapes")
decode_compile_shapes = _get_int_pair_list_config("decode_compile_shapes")
decode_batch_bucket_frames = _get_int_list_config("decode_batch_bucket_frames")
if decode_batch_bucket_frames is not None:
self._decode_batch_bucket_frames = decode_batch_bucket_frames
decode_batch_max_size = _get_int_config("decode_batch_max_size", self._decode_batch_max_size)
if decode_batch_max_size < 0:
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config decode_batch_max_size={decode_batch_max_size}")
self._decode_batch_max_size = decode_batch_max_size
decode_variable_chunk_batch_min_frames = _get_int_config(
"decode_variable_chunk_batch_min_frames",
self._decode_variable_chunk_batch_min_frames,
)
if decode_variable_chunk_batch_min_frames < 0:
raise ValueError(
"Invalid Qwen3-TTS Code2Wav config "
f"decode_variable_chunk_batch_min_frames={decode_variable_chunk_batch_min_frames}"
)
self._decode_variable_chunk_batch_min_frames = decode_variable_chunk_batch_min_frames
decode_enable_tf32 = _get_bool_config("decode_enable_tf32", False)
else:
decode_cudagraph_capture_sizes = None
decode_cudagraph_batch_sizes = None
decode_cudagraph_extra_capture_shapes = None
decode_compile_shapes = None
decode_enable_tf32 = False
if decode_enable_tf32 and device.type == "cuda":
# PyTorch exposes TF32 controls as process-wide CUDA backend
# switches. This opt-in is intended for deployments where
# Code2Wav runs in its own Stage1 worker process.
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
torch.set_float32_matmul_precision("high")
logger.info(
"Qwen3-TTS Code2Wav TF32 enabled process-wide: "
"matmul.allow_tf32=%s cudnn.allow_tf32=%s float32_matmul_precision=%s",
torch.backends.cuda.matmul.allow_tf32,
torch.backends.cudnn.allow_tf32,
torch.get_float32_matmul_precision(),
)
if hasattr(self.decoder, "enable_cudagraph") and device.type == "cuda":
try:
@@ -339,6 +599,10 @@ class Qwen3TTSCode2Wav(nn.Module):
)
self.decoder.enable_cudagraph(
capture_sizes=decode_cudagraph_capture_sizes,
capture_batch_sizes=decode_cudagraph_batch_sizes,
extra_capture_shapes=decode_cudagraph_extra_capture_shapes,
compile_shapes=decode_compile_shapes,
device=device,
codec_chunk_frames=codec_chunk_frames,
codec_left_context_frames=codec_left_context_frames,
@@ -37,6 +37,12 @@ from .qwen3_tts_tokenizer import Qwen3TTSTokenizer
logger = init_logger(__name__)
_TRAILING_TEXT_COMPACT_MIN_FRAMES = 64
_PRECOMPUTED_REF_CODE_KEY = "precomputed_ref"
_NORMALIZED_REF_AUDIO_KEY = "_qwen3_tts_normalized_ref_audio"
_PRECOMPUTED_TEXT_IDS_KEY = "_qwen3_tts_text_ids"
_PRECOMPUTED_REF_IDS_KEY = "_qwen3_tts_ref_ids"
# ---------------------------------------------------------------------------
# Components ported from the HuggingFace Qwen3-TTS reference implementation.
@@ -259,6 +265,8 @@ def mel_spectrogram(
fmin: int,
fmax: int | None = None,
center: bool = False,
mel_basis: torch.Tensor | None = None,
hann_window: torch.Tensor | None = None,
) -> torch.Tensor:
"""Calculate mel spectrogram of an input signal using torchaudio mel filterbank and torch STFT."""
if torch.min(y) < -1.0:
@@ -266,14 +274,20 @@ def mel_spectrogram(
if torch.max(y) > 1.0:
logger.warning("Max value of input waveform signal is %s", torch.max(y))
device = y.device
mel_basis = mel_filter_bank(
sr=sampling_rate,
n_fft=n_fft,
n_mels=num_mels,
fmin=fmin,
fmax=fmax,
).to(device)
hann_window = torch.hann_window(win_size).to(device)
if mel_basis is None:
mel_basis = mel_filter_bank(
sr=sampling_rate,
n_fft=n_fft,
n_mels=num_mels,
fmin=fmin,
fmax=fmax,
).to(device)
elif mel_basis.device != device:
mel_basis = mel_basis.to(device)
if hann_window is None:
hann_window = torch.hann_window(win_size, device=device)
elif hann_window.device != device:
hann_window = hann_window.to(device)
padding = (n_fft - hop_size) // 2
y = torch.nn.functional.pad(y.unsqueeze(1), (padding, padding), mode="reflect").squeeze(1)
spec = torch.stft(
@@ -560,6 +574,14 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
span_len = int(input_ids.shape[0])
if span_len <= 0:
return input_ids, input_embeds if input_embeds is not None else self.embed_input_ids(input_ids), {}
is_prefill_raw = info_dict.get("_omni_is_prefill")
if isinstance(is_prefill_raw, bool):
is_prefill = is_prefill_raw
else:
try:
is_prefill = int(info_dict["_omni_num_computed_tokens"]) < int(info_dict["_omni_prompt_len"])
except Exception:
is_prefill = span_len > 1
text_list = info_dict.get("text")
if not isinstance(text_list, list) or not text_list or not text_list[0]:
@@ -576,7 +598,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
else:
codec_streaming = task_type == "Base"
if span_len > 1:
if is_prefill:
# Prefill (prompt embeddings)
prompt_embeds_cpu = embed.get("prefill")
tts_pad_embed_cpu = embed.get("tts_pad")
@@ -600,7 +622,11 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
"tts_pad": tts_pad_embed.detach(),
},
"hidden_states": {"trailing_text": tailing_text_hidden.detach()},
"meta": {"talker_prefill_offset": 0, "codec_streaming": codec_streaming},
"meta": {
"talker_prefill_offset": 0,
"talker_text_offset": 0,
"codec_streaming": codec_streaming,
},
}
if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0:
info_update.setdefault("codes", {})["ref"] = ref_code.detach().to("cpu").contiguous()
@@ -634,7 +660,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
take = torch.cat([take, pad_rows], dim=0)
prompt_embeds = take.to(device=input_ids.device, dtype=torch.bfloat16)
info_update = {
"meta": {"talker_prefill_offset": int(offset + span_len), "codec_streaming": codec_streaming}
"meta": {
"talker_prefill_offset": int(offset + span_len),
"codec_streaming": codec_streaming,
}
}
# When inputs_embeds is set, token ids are ignored by the model but must stay in-vocab for vLLM bookkeeping.
@@ -658,12 +687,37 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
tts_pad_embed = tts_pad_embed_buf.to(device=input_ids.device, dtype=torch.bfloat16).reshape(1, -1)
tail = hs.get("trailing_text")
if isinstance(tail, torch.Tensor) and tail.ndim == 2 and tail.shape[0] > 0:
text_step = tail[:1].to(device=input_ids.device, dtype=torch.bfloat16).reshape(1, -1)
new_tail = tail[1:] if tail.shape[0] > 1 else tail[:0]
text_offset = max(0, int(meta.get("talker_text_offset", 0) or 0))
trailing_text_update = None
if isinstance(tail, torch.Tensor) and tail.ndim == 2:
tail_len = int(tail.shape[0])
if text_offset < tail_len:
text_step = (
tail[text_offset : text_offset + 1]
.to(
device=input_ids.device,
dtype=torch.bfloat16,
)
.reshape(1, -1)
)
next_text_offset = text_offset + 1
should_compact_tail = next_text_offset >= tail_len or (
next_text_offset >= _TRAILING_TEXT_COMPACT_MIN_FRAMES and next_text_offset * 2 >= tail_len
)
if should_compact_tail:
if next_text_offset >= tail_len:
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
else:
trailing_text_update = tail[next_text_offset:].contiguous()
next_text_offset = 0
else:
text_step = tts_pad_embed
next_text_offset = 0
if tail.numel() > 0:
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
else:
text_step = tts_pad_embed
new_tail = tail if isinstance(tail, torch.Tensor) else torch.empty((0, tts_pad_embed.shape[-1]))
next_text_offset = text_offset
last_hidden = hs.get("last")
if not isinstance(last_hidden, torch.Tensor):
@@ -677,12 +731,127 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
inputs_embeds_out = last_id_hidden.reshape(1, -1)
info_update = {
"hidden_states": {"trailing_text": new_tail},
"mtp_inputs": (past_hidden, text_step),
"meta": {"codec_streaming": codec_streaming},
"meta": {
"talker_text_offset": int(next_text_offset),
"codec_streaming": codec_streaming,
},
}
if trailing_text_update is not None:
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
return input_ids, inputs_embeds_out, info_update
def preprocess_decode_batch(
self,
*,
input_ids: torch.Tensor,
req_infos: list[dict[str, Any]],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, list[dict[str, Any]]]:
"""Batch the decode-only preprocess path for Qwen3-TTS.
This mirrors the scalar decode branch in ``preprocess()``, but performs
the token embedding lookup once for the whole decode batch.
"""
input_ids_flat = input_ids.reshape(-1)
if int(input_ids_flat.numel()) != len(req_infos):
raise ValueError(
f"preprocess_decode_batch expected {len(req_infos)} input ids, got {int(input_ids_flat.numel())}"
)
device = input_ids_flat.device
dtype = torch.bfloat16
past_hidden_list: list[torch.Tensor] = []
text_step_list: list[torch.Tensor] = []
updates: list[dict[str, Any]] = []
for info_dict in req_infos:
additional_information = info_dict.get("additional_information")
if isinstance(additional_information, dict):
merged: dict[str, Any] = {k: v for k, v in info_dict.items() if k != "additional_information"}
for k, v in additional_information.items():
merged.setdefault(k, v)
info_dict = merged
payload: OmniPayload = info_dict
embed = payload.get("embed", {})
hs = payload.get("hidden_states", {})
meta = payload.get("meta", {})
text_list = info_dict.get("text")
if not isinstance(text_list, list) or not text_list or not text_list[0]:
raise ValueError("Missing additional_information.text for Qwen3-TTS AR talker.")
task_type = (info_dict.get("task_type") or ["CustomVoice"])[0]
codec_streaming_val = meta.get("codec_streaming")
if isinstance(codec_streaming_val, list):
codec_streaming_raw = codec_streaming_val[0] if codec_streaming_val else None
else:
codec_streaming_raw = codec_streaming_val
if isinstance(codec_streaming_raw, bool):
codec_streaming = codec_streaming_raw
else:
codec_streaming = task_type == "Base"
tts_pad_embed_buf = embed.get("tts_pad")
if not isinstance(tts_pad_embed_buf, torch.Tensor):
raise RuntimeError("Missing `tts_pad_embed` in additional_information; prefill must run first.")
tts_pad_embed = tts_pad_embed_buf.to(device=device, dtype=dtype).reshape(1, -1)
tail = hs.get("trailing_text")
text_offset = max(0, int(meta.get("talker_text_offset", 0) or 0))
trailing_text_update = None
if isinstance(tail, torch.Tensor) and tail.ndim == 2:
tail_len = int(tail.shape[0])
if text_offset < tail_len:
text_step = tail[text_offset : text_offset + 1].to(device=device, dtype=dtype).reshape(1, -1)
next_text_offset = text_offset + 1
should_compact_tail = next_text_offset >= tail_len or (
next_text_offset >= _TRAILING_TEXT_COMPACT_MIN_FRAMES and next_text_offset * 2 >= tail_len
)
if should_compact_tail:
if next_text_offset >= tail_len:
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
else:
trailing_text_update = tail[next_text_offset:].contiguous()
next_text_offset = 0
else:
text_step = tts_pad_embed
next_text_offset = 0
if tail.numel() > 0:
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
else:
text_step = tts_pad_embed
next_text_offset = text_offset
last_hidden = hs.get("last")
if not isinstance(last_hidden, torch.Tensor):
raise RuntimeError("Missing hidden_states['last'] in additional_information; postprocess must run.")
past_hidden_list.append(last_hidden.to(device=device, dtype=dtype).reshape(1, -1))
text_step_list.append(text_step)
info_update: dict[str, Any] = {
"meta": {
"talker_text_offset": int(next_text_offset),
"codec_streaming": codec_streaming,
},
}
if trailing_text_update is not None:
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
updates.append(info_update)
inputs_embeds_out = self.embed_input_ids(input_ids_flat.reshape(-1, 1).to(torch.long)).to(
device=device,
dtype=dtype,
)
inputs_embeds_out = inputs_embeds_out.reshape(len(req_infos), -1)
return (
input_ids_flat,
inputs_embeds_out,
torch.cat(past_hidden_list, dim=0),
torch.cat(text_step_list, dim=0),
updates,
)
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
# Keep the last token hidden for the next decode step's code predictor.
# Stays on GPU - gpu_resident_buffer_keys avoids the CPU round-trip.
@@ -1121,9 +1290,21 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
wav = resampler.resample(wav.astype(np.float32), orig_sr=int(sr))
sr = target_sr
# Follow official implementation: mel_spectrogram expects 24kHz.
# Follow official implementation: mel_spectrogram expects 24kHz. Move
# the waveform first so STFT/mel computation stays on the model device
# instead of materializing a CPU mel tensor and copying it per request.
wav_tensor = torch.from_numpy(wav).to(device=dev, dtype=torch.float32).unsqueeze(0)
mel_basis, hann_window = self._get_speaker_mel_buffers(
device=dev,
sampling_rate=24000,
n_fft=1024,
num_mels=128,
win_size=1024,
fmin=0,
fmax=12000,
)
mels = mel_spectrogram(
torch.from_numpy(wav).unsqueeze(0),
wav_tensor,
n_fft=1024,
num_mels=128,
sampling_rate=24000,
@@ -1131,10 +1312,43 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
win_size=1024,
fmin=0,
fmax=12000,
mel_basis=mel_basis,
hann_window=hann_window,
).transpose(1, 2)
spk = self.speaker_encoder(mels.to(dev, dtype=torch.bfloat16))[0]
spk = self.speaker_encoder(mels.to(dtype=torch.bfloat16))[0]
return spk.to(dtype=torch.bfloat16)
def _get_speaker_mel_buffers(
self,
*,
device: torch.device,
sampling_rate: int,
n_fft: int,
num_mels: int,
win_size: int,
fmin: int,
fmax: int | None,
) -> tuple[torch.Tensor, torch.Tensor]:
cache = getattr(self, "_speaker_mel_buffer_cache", None)
if cache is None:
cache = {}
self._speaker_mel_buffer_cache = cache
key = (str(device), int(sampling_rate), int(n_fft), int(num_mels), int(win_size), int(fmin), fmax)
cached = cache.get(key)
if cached is not None:
return cached
mel_basis = mel_filter_bank(
sr=int(sampling_rate),
n_fft=int(n_fft),
n_mels=int(num_mels),
fmin=int(fmin),
fmax=fmax,
).to(device)
hann_window = torch.hann_window(int(win_size), device=device)
cached = (mel_basis, hann_window)
cache[key] = cached
return cached
def _ensure_speech_tokenizer_loaded(self) -> Qwen3TTSTokenizer:
if self._speech_tokenizer is not None:
return self._speech_tokenizer
@@ -1182,6 +1396,198 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
return ref_code.to(device=next(self.parameters()).device, dtype=torch.long)
raise ValueError("SpeechTokenizer.encode did not return audio_codes tensor")
@staticmethod
def _first_value(value: object, default: object = None) -> object:
if isinstance(value, list):
return value[0] if value else default
return value if value is not None else default
@staticmethod
def _coerce_ref_code_tensor(value: object, *, device: torch.device) -> torch.Tensor | None:
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
if value is None:
return None
if isinstance(value, torch.Tensor):
ref_code = value
elif isinstance(value, np.ndarray):
ref_code = torch.from_numpy(value)
elif isinstance(value, list) and value:
ref_code = torch.as_tensor(value, dtype=torch.long)
else:
return None
if ref_code.ndim == 3:
ref_code = ref_code[0]
if ref_code.ndim != 2 or ref_code.numel() == 0:
return None
return ref_code.to(device=device, dtype=torch.long).contiguous()
@staticmethod
def _coerce_token_ids(value: object, *, device: torch.device) -> torch.Tensor | None:
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
if value is None:
return None
if isinstance(value, torch.Tensor):
ids = value
elif isinstance(value, np.ndarray):
ids = torch.from_numpy(value)
elif isinstance(value, list) and value and all(isinstance(v, (int, np.integer)) for v in value):
ids = torch.tensor(value, dtype=torch.long)
else:
return None
if ids.ndim == 1:
ids = ids.unsqueeze(0)
if ids.ndim != 2 or ids.numel() == 0:
return None
return ids.to(device=device, dtype=torch.long).contiguous()
@staticmethod
def _split_ref_code_batch(ref_codes: object, expected: int) -> list[object] | None:
if isinstance(ref_codes, torch.Tensor):
if ref_codes.ndim == 3 and int(ref_codes.shape[0]) == expected:
return [ref_codes[i] for i in range(expected)]
if expected == 1:
return [ref_codes]
return None
if isinstance(ref_codes, (list, tuple)) and len(ref_codes) == expected:
return list(ref_codes)
return None
@staticmethod
def _voice_clone_prompt_dict(raw: object) -> dict[str, object] | None:
raw = Qwen3TTSTalkerForConditionalGeneration._first_value(raw)
if isinstance(raw, dict):
return raw
if isinstance(raw, list) and raw and isinstance(raw[0], dict):
return raw[0]
return None
@staticmethod
def _has_ref_code_like(value: object) -> bool:
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
if isinstance(value, torch.Tensor):
return value.numel() > 0 and value.ndim in (2, 3)
if isinstance(value, np.ndarray):
return value.size > 0 and value.ndim in (2, 3)
if isinstance(value, list):
return bool(value)
return False
@staticmethod
def _needs_initial_prompt_preprocess(info_dict: dict[str, Any]) -> bool:
embed = info_dict.get("embed")
return not (isinstance(embed, dict) and "prefill" in embed)
def _needs_batched_ref_code(self, info_dict: dict[str, Any]) -> bool:
if not self._needs_initial_prompt_preprocess(info_dict):
return False
if self._first_value(info_dict.get("task_type"), "CustomVoice") != "Base":
return False
codes = info_dict.get("codes")
if isinstance(codes, dict) and self._has_ref_code_like(codes.get(_PRECOMPUTED_REF_CODE_KEY)):
return False
voice_clone_prompt = self._voice_clone_prompt_dict(info_dict.get("voice_clone_prompt"))
if voice_clone_prompt is not None:
if self._has_ref_code_like(voice_clone_prompt.get("ref_code")):
return False
icl_flag = self._first_value(voice_clone_prompt.get("icl_mode"))
if isinstance(icl_flag, bool) and not icl_flag:
return False
xvec_only = self._first_value(info_dict.get("x_vector_only_mode"), False)
if isinstance(xvec_only, bool) and xvec_only:
return False
ref_audio_list = info_dict.get("ref_audio")
return isinstance(ref_audio_list, list) and bool(ref_audio_list)
@torch.inference_mode()
def preprocess_batch(
self,
*,
req_ids: list[str],
model_intermediate_buffer: dict[str, dict[str, Any]],
device: torch.device,
) -> None:
"""Batch Base voice-clone ref-audio codec extraction for current prefill requests."""
pending_text: list[tuple[dict[str, Any], str]] = []
pending_ref_text: list[tuple[dict[str, Any], str]] = []
groups: dict[int, list[tuple[dict[str, Any], np.ndarray, int]]] = {}
for req_id in req_ids:
info_dict = model_intermediate_buffer.get(req_id)
if not isinstance(info_dict, dict):
continue
if (
self._needs_initial_prompt_preprocess(info_dict)
and self._first_value(info_dict.get("task_type"), "CustomVoice") == "Base"
):
if _PRECOMPUTED_TEXT_IDS_KEY not in info_dict:
text = self._first_value(info_dict.get("text"), "")
if isinstance(text, str):
pending_text.append((info_dict, self._build_assistant_text(text)))
if _PRECOMPUTED_REF_IDS_KEY not in info_dict:
ref_text = self._first_value(info_dict.get("ref_text"))
if isinstance(ref_text, str) and ref_text.strip():
pending_ref_text.append((info_dict, self._build_ref_text(ref_text)))
if not self._needs_batched_ref_code(info_dict):
continue
ref_audio_list = info_dict.get("ref_audio")
if not isinstance(ref_audio_list, list) or not ref_audio_list:
continue
try:
wav, sr = self._normalize_ref_audio(ref_audio_list[0])
except Exception:
# Keep the original per-request path responsible for surfacing
# invalid ref_audio errors with its existing messages.
continue
info_dict[_NORMALIZED_REF_AUDIO_KEY] = (wav, sr)
groups.setdefault(int(sr), []).append((info_dict, wav, int(sr)))
if pending_text or pending_ref_text:
tok_text = self._get_tokenizer()
for items, key in (
(pending_text, _PRECOMPUTED_TEXT_IDS_KEY),
(pending_ref_text, _PRECOMPUTED_REF_IDS_KEY),
):
if not items:
continue
try:
tokenized = tok_text([text for _, text in items], padding=False)
except Exception as exc:
logger.debug("Qwen3-TTS batched text tokenization failed; falling back to serial path: %s", exc)
continue
input_ids = tokenized.get("input_ids") if isinstance(tokenized, dict) else None
if not isinstance(input_ids, list) or len(input_ids) != len(items):
continue
for (info_dict, _), ids in zip(items, input_ids, strict=True):
if isinstance(ids, list) and ids:
info_dict[key] = torch.tensor(ids, dtype=torch.long)
groups = {sr: items for sr, items in groups.items() if len(items) >= 2}
if not groups:
return
tok = self._ensure_speech_tokenizer_loaded()
for sr, items in groups.items():
wavs = [wav for _, wav, _ in items]
try:
enc = tok.encode(wavs, sr=int(sr), return_dict=True)
ref_codes = self._split_ref_code_batch(getattr(enc, "audio_codes", None), len(items))
except Exception as exc:
logger.debug("Qwen3-TTS batched ref_code encode failed; falling back to serial path: %s", exc)
continue
if ref_codes is None:
continue
for (info_dict, wav, item_sr), ref_code in zip(items, ref_codes, strict=True):
ref_code_t = self._coerce_ref_code_tensor(ref_code, device=device)
if ref_code_t is None:
continue
info_dict.setdefault("codes", {})[_PRECOMPUTED_REF_CODE_KEY] = ref_code_t
info_dict[_NORMALIZED_REF_AUDIO_KEY] = (wav, item_sr)
def _generate_icl_prompt(
self,
*,
@@ -1256,9 +1662,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
# Text ids for assistant template (always).
tok = self._get_tokenizer()
input_ids = tok(self._build_assistant_text(text), return_tensors="pt", padding=False)["input_ids"].to(
device=next(self.parameters()).device
)
dev = next(self.parameters()).device
input_ids = self._coerce_token_ids(info_dict.pop(_PRECOMPUTED_TEXT_IDS_KEY, None), device=dev)
if input_ids is None:
input_ids = tok(self._build_assistant_text(text), return_tensors="pt", padding=False)["input_ids"].to(
device=dev
)
# Optional instruct prefix.
instruct = (info_dict.get("instruct") or [""])[0]
@@ -1362,6 +1771,26 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
xvec_only = bool((info_dict.get("x_vector_only_mode") or [False])[0])
in_context_mode = not xvec_only
voice_clone_prompt = _normalize_voice_clone_prompt(info_dict.get("voice_clone_prompt"))
ref_audio_wav: np.ndarray | None = None
ref_audio_sr: int | None = None
def _get_ref_audio() -> tuple[np.ndarray, int]:
nonlocal ref_audio_wav, ref_audio_sr
if ref_audio_wav is None or ref_audio_sr is None:
normalized_ref_audio = info_dict.pop(_NORMALIZED_REF_AUDIO_KEY, None)
if (
isinstance(normalized_ref_audio, tuple)
and len(normalized_ref_audio) == 2
and isinstance(normalized_ref_audio[0], np.ndarray)
):
ref_audio_wav = normalized_ref_audio[0]
ref_audio_sr = int(normalized_ref_audio[1])
return ref_audio_wav, ref_audio_sr
ref_audio_list = info_dict.get("ref_audio")
if not isinstance(ref_audio_list, list) or not ref_audio_list:
raise ValueError("Base requires `ref_audio`.")
ref_audio_wav, ref_audio_sr = self._normalize_ref_audio(ref_audio_list[0])
return ref_audio_wav, ref_audio_sr
# Speaker cache: only for uploaded (named) speakers
_speaker_cache_key = None
@@ -1413,12 +1842,15 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
ref_code_t = ref_code_t[0]
ref_code_t = ref_code_t.to(device=input_ids.device, dtype=torch.long)
ref_code_len = int(ref_code_t.shape[0])
elif in_context_mode:
else:
codes = info_dict.get("codes")
precomputed_ref_code = codes.get(_PRECOMPUTED_REF_CODE_KEY) if isinstance(codes, dict) else None
ref_code_t = self._coerce_ref_code_tensor(precomputed_ref_code, device=input_ids.device)
if isinstance(ref_code_t, torch.Tensor):
ref_code_len = int(ref_code_t.shape[0])
if ref_code_t is None and in_context_mode:
# Compute ref_code from ref_audio if not provided.
ref_audio_list = info_dict.get("ref_audio")
if not isinstance(ref_audio_list, list) or not ref_audio_list:
raise ValueError("Base requires `ref_audio`.")
wav_np, sr = self._normalize_ref_audio(ref_audio_list[0])
wav_np, sr = _get_ref_audio()
ref_code_t = self._encode_ref_audio_to_code(wav_np, sr).to(device=input_ids.device)
ref_code_len = int(ref_code_t.shape[0])
if isinstance(ref_code_t, torch.Tensor):
@@ -1437,10 +1869,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
# Plain list/array from API (survived msgspec IPC serialization).
speaker_embed = torch.tensor(spk, dtype=torch.bfloat16, device=input_ids.device).view(1, 1, -1)
else:
ref_audio_list = info_dict.get("ref_audio")
if not isinstance(ref_audio_list, list) or not ref_audio_list:
raise ValueError("Base requires `ref_audio`.")
wav_np, sr = self._normalize_ref_audio(ref_audio_list[0])
wav_np, sr = _get_ref_audio()
speaker_embed = self._extract_speaker_embedding(wav_np, sr).view(1, 1, -1)
# Cache miss: store extraction result
@@ -1467,7 +1896,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
if in_context_mode:
# Prefer explicit tokenized `ref_ids` if provided (matches official signature).
ref_ids = _to_long_tensor(info_dict.get("ref_ids"), device=input_ids.device)
ref_ids = self._coerce_token_ids(info_dict.pop(_PRECOMPUTED_REF_IDS_KEY, None), device=input_ids.device)
if ref_ids is None:
ref_ids = _to_long_tensor(info_dict.get("ref_ids"), device=input_ids.device)
if ref_ids is None and voice_clone_prompt is not None:
ref_ids = _to_long_tensor(
voice_clone_prompt.get("ref_ids") or voice_clone_prompt.get("ref_id"), device=input_ids.device
@@ -852,6 +852,9 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
def enable_cudagraph(
self,
capture_sizes: list[int] | None = None,
capture_batch_sizes: list[int] | None = None,
extra_capture_shapes: list[tuple[int, int]] | None = None,
compile_shapes: list[tuple[int, int]] | None = None,
device: torch.device | None = None,
codec_chunk_frames: int = 0,
codec_left_context_frames: int = 0,
@@ -869,6 +872,9 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
self._cudagraph_wrapper = CUDAGraphDecoderWrapper(
decoder=self,
capture_sizes=capture_sizes,
capture_batch_sizes=capture_batch_sizes,
extra_capture_shapes=extra_capture_shapes,
compile_shapes=compile_shapes,
num_quantizers=self.config.num_quantizers,
enabled=True,
)
@@ -882,8 +888,11 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
)
self._cudagraph_enabled = True
logger.info(
"CUDA Graph enabled for decoder: seq_lens=%s",
"CUDA Graph enabled for decoder: batch_sizes=%s seq_lens=%s extra_shapes=%s compile_shapes=%s",
self._cudagraph_wrapper.capture_batch_sizes,
self._cudagraph_wrapper.capture_sizes,
self._cudagraph_wrapper.extra_capture_shapes,
self._cudagraph_wrapper.compile_shapes,
)
def disable_cudagraph(self):
@@ -925,6 +934,35 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
start_index = end_index
return torch.cat(wavs, dim=-1)
def batched_chunked_decode(
self,
codes,
lengths,
chunk_size=300,
left_context_size=25,
max_batch_size=0,
):
if self._cudagraph_enabled and self._cudagraph_wrapper is not None:
return self._cudagraph_wrapper.batched_chunked_decode_with_cudagraph(
codes,
lengths,
chunk_size=chunk_size,
left_context_size=left_context_size,
max_batch_size=max_batch_size,
)
from ..cuda_graph_decoder_wrapper import _batched_chunked_decode
return _batched_chunked_decode(
codes,
lengths,
decode_fn=self,
total_upsample=self.total_upsample,
chunk_size=chunk_size,
left_context_size=left_context_size,
max_batch_size=max_batch_size,
)
class Qwen3TTSTokenizerV2Encoder(MimiModel):
def __init__(self, config: MimiConfig):