[TTS][Perf] Optimize Qwen3-TTS high-concurrency serving (#3662)
Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
14 files changed
+2703
-147
No files matched your search
@@ -467,10 +467,26 @@ class CodePredictorWrapper(nn.Module):
|
||||
self._model_dtype: torch.dtype | None = None
|
||||
self._compiled_model_fwd = None
|
||||
self._bucket_sizes: list[int] = []
|
||||
self._bucket_pos_ids: dict[int, torch.Tensor] = {}
|
||||
self._bucket_pos_ids: dict[int | tuple[int, int], torch.Tensor] = {}
|
||||
self._lm_heads_list: list[nn.Module] | None = None
|
||||
self._codec_embeds_list: list[nn.Module] | None = None
|
||||
self._device_graphs: dict[int, tuple] = {} # (graph, static_output) per bucket
|
||||
self._device_graphs: dict[int | tuple[int, int], tuple] = {} # (graph, static_output) per bucket
|
||||
prefix_graph_cfg = self._stage_connector_extra_config(vllm_config)
|
||||
prefix_graphs_requested = self._parse_bool_config(prefix_graph_cfg.get("code_predictor_prefix_graphs"))
|
||||
is_npu = current_omni_platform.is_npu()
|
||||
self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu
|
||||
if prefix_graphs_requested and not self._prefix_graphs_enabled:
|
||||
logger.info_once(
|
||||
"code_predictor: prefix CUDA graphs requested but disabled because use_cuda_graphs=%s is_npu=%s",
|
||||
wrapper_config.use_cuda_graphs,
|
||||
is_npu,
|
||||
)
|
||||
self._prefix_graph_buckets = self._parse_positive_int_set(
|
||||
prefix_graph_cfg.get("code_predictor_prefix_graph_buckets")
|
||||
)
|
||||
self._prefix_graph_seq_lens = self._parse_positive_int_set(
|
||||
prefix_graph_cfg.get("code_predictor_prefix_graph_seq_lens")
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.model.get_input_embeddings()
|
||||
@@ -546,6 +562,56 @@ class CodePredictorWrapper(nn.Module):
|
||||
return bucket
|
||||
return bsz
|
||||
|
||||
@staticmethod
|
||||
def _stage_connector_extra_config(vllm_config: VllmConfig) -> dict:
|
||||
model_cfg = getattr(vllm_config, "model_config", None)
|
||||
connector_cfg = getattr(model_cfg, "stage_connector_config", None)
|
||||
if isinstance(connector_cfg, dict):
|
||||
extra_cfg = connector_cfg.get("extra", connector_cfg)
|
||||
else:
|
||||
extra_cfg = getattr(connector_cfg, "extra", None)
|
||||
return extra_cfg if isinstance(extra_cfg, dict) else {}
|
||||
|
||||
@staticmethod
|
||||
def _parse_bool_config(value: object) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in ("1", "true", "yes", "on")
|
||||
if isinstance(value, int):
|
||||
return bool(value)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_positive_int_set(value: object) -> set[int]:
|
||||
if value is None:
|
||||
return set()
|
||||
if isinstance(value, str):
|
||||
raw_values = [item.strip() for item in value.replace(";", ",").split(",") if item.strip()]
|
||||
elif isinstance(value, int):
|
||||
raw_values = [value]
|
||||
else:
|
||||
try:
|
||||
raw_values = list(value)
|
||||
except TypeError as exc:
|
||||
raise ValueError(f"Invalid positive int config value {value!r}") from exc
|
||||
values: set[int] = set()
|
||||
for item in raw_values:
|
||||
try:
|
||||
parsed = int(item)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid positive int config value {item!r}") from exc
|
||||
if parsed > 0:
|
||||
values.add(parsed)
|
||||
return values
|
||||
|
||||
def _prefix_seq_lens(self, max_seq: int) -> list[int]:
|
||||
all_seq_lens = list(range(2, max_seq))
|
||||
if not self._prefix_graph_seq_lens:
|
||||
return all_seq_lens
|
||||
allowed = set(all_seq_lens)
|
||||
return sorted(seq_len for seq_len in self._prefix_graph_seq_lens if seq_len in allowed)
|
||||
|
||||
def _warmup_buckets(self) -> None:
|
||||
"""Warmup power-of-2 batch-size buckets to front-load Inductor compilation."""
|
||||
max_bsz = self._vllm_config.scheduler_config.max_num_seqs
|
||||
@@ -562,12 +628,44 @@ class CodePredictorWrapper(nn.Module):
|
||||
self._ensure_buffers(device, self._model_dtype, max(self._bucket_sizes))
|
||||
proj_buf = self._proj_buf
|
||||
|
||||
for bsz in self._bucket_sizes:
|
||||
pos_ids = torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
|
||||
self._bucket_pos_ids[bsz] = pos_ids
|
||||
for _ in range(3):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
|
||||
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
|
||||
if self._prefix_graphs_enabled:
|
||||
prefix_seq_lens = self._prefix_seq_lens(max_seq)
|
||||
needs_full_graph = set(prefix_seq_lens) != set(range(2, max_seq))
|
||||
for bsz in self._bucket_sizes:
|
||||
capture_prefixes = not self._prefix_graph_buckets or bsz in self._prefix_graph_buckets
|
||||
if not capture_prefixes or needs_full_graph:
|
||||
pos_ids = (
|
||||
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
|
||||
)
|
||||
self._bucket_pos_ids[bsz] = pos_ids
|
||||
for _ in range(3):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
|
||||
if capture_prefixes:
|
||||
for seq_len in prefix_seq_lens:
|
||||
pos_ids = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.long)
|
||||
.unsqueeze(0)
|
||||
.expand(bsz, -1)
|
||||
.contiguous()
|
||||
)
|
||||
self._bucket_pos_ids[(bsz, seq_len)] = pos_ids
|
||||
for _ in range(2):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :seq_len, :], pos_ids)
|
||||
logger.info(
|
||||
"code_predictor: prefix warmup done for buckets %s prefix_buckets=%s seq_lens=%s",
|
||||
self._bucket_sizes,
|
||||
sorted(self._prefix_graph_buckets) if self._prefix_graph_buckets else "all",
|
||||
prefix_seq_lens,
|
||||
)
|
||||
else:
|
||||
for bsz in self._bucket_sizes:
|
||||
pos_ids = (
|
||||
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(bsz, -1).contiguous()
|
||||
)
|
||||
self._bucket_pos_ids[bsz] = pos_ids
|
||||
for _ in range(3):
|
||||
self._compiled_model_fwd(proj_buf[:bsz, :max_seq, :], pos_ids)
|
||||
logger.info("code_predictor: warmup done for buckets %s", self._bucket_sizes)
|
||||
|
||||
def _capture_cuda_graphs(self) -> None:
|
||||
"""Capture a CUDA graph per bucket using vLLM's global graph pool."""
|
||||
@@ -577,17 +675,50 @@ class CodePredictorWrapper(nn.Module):
|
||||
max_seq = self._num_groups + 1
|
||||
proj_buf = self._proj_buf
|
||||
|
||||
for bsz in self._bucket_sizes:
|
||||
static_input = proj_buf[:bsz, :max_seq, :]
|
||||
pos_ids = self._bucket_pos_ids[bsz]
|
||||
if self._prefix_graphs_enabled:
|
||||
prefix_seq_lens = self._prefix_seq_lens(max_seq)
|
||||
needs_full_graph = set(prefix_seq_lens) != set(range(2, max_seq))
|
||||
for bsz in self._bucket_sizes:
|
||||
capture_prefixes = not self._prefix_graph_buckets or bsz in self._prefix_graph_buckets
|
||||
if not capture_prefixes or needs_full_graph:
|
||||
static_input = proj_buf[:bsz, :max_seq, :]
|
||||
pos_ids = self._bucket_pos_ids[bsz]
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
|
||||
self._device_graphs[bsz] = (g, static_output)
|
||||
self._device_graphs[bsz] = (g, static_output)
|
||||
|
||||
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
|
||||
if capture_prefixes:
|
||||
for seq_len in prefix_seq_lens:
|
||||
static_input = proj_buf[:bsz, :seq_len, :]
|
||||
pos_ids = self._bucket_pos_ids[(bsz, seq_len)]
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
|
||||
self._device_graphs[(bsz, seq_len)] = (g, static_output)
|
||||
|
||||
logger.info(
|
||||
"code_predictor: captured prefix CUDA graphs for buckets %s prefix_buckets=%s seq_lens=%s",
|
||||
self._bucket_sizes,
|
||||
sorted(self._prefix_graph_buckets) if self._prefix_graph_buckets else "all",
|
||||
prefix_seq_lens,
|
||||
)
|
||||
else:
|
||||
for bsz in self._bucket_sizes:
|
||||
static_input = proj_buf[:bsz, :max_seq, :]
|
||||
pos_ids = self._bucket_pos_ids[bsz]
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g, pool=pool):
|
||||
static_output = self._compiled_model_fwd(static_input, pos_ids)
|
||||
|
||||
self._device_graphs[bsz] = (g, static_output)
|
||||
|
||||
logger.info("code_predictor: captured CUDA graphs for buckets %s", self._bucket_sizes)
|
||||
|
||||
def _capture_npu_graphs(self) -> None:
|
||||
"""Capture an NPU graph per bucket using torch_npu's NPUGraph."""
|
||||
@@ -650,16 +781,6 @@ class CodePredictorWrapper(nn.Module):
|
||||
proj_buf[:bsz, 0, :] = projection(last_talker_hidden.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
proj_buf[:bsz, 1, :] = projection(layer0_embed.reshape(bsz, 1, -1).to(dtype)).reshape(bsz, -1)
|
||||
|
||||
# Get pre-computed pos_ids for this bucket
|
||||
full_pos_ids = self._bucket_pos_ids.get(padded_bsz)
|
||||
if full_pos_ids is None:
|
||||
full_pos_ids = (
|
||||
torch.arange(max_seq, device=device, dtype=torch.long).unsqueeze(0).expand(padded_bsz, -1).contiguous()
|
||||
)
|
||||
|
||||
# Use captured device graph if available, otherwise call compiled fn.
|
||||
device_graph_entry = self._device_graphs.get(padded_bsz)
|
||||
|
||||
# Prepare sampling parameters
|
||||
stored_mode = self._wrapper_config.sampling_mode == "stored"
|
||||
if stored_mode:
|
||||
@@ -683,12 +804,31 @@ class CodePredictorWrapper(nn.Module):
|
||||
|
||||
# Autoregressive loop: predict layers 1..G-1
|
||||
for step in range(1, num_groups):
|
||||
graph_key: int | tuple[int, int] = padded_bsz
|
||||
seq_len = max_seq
|
||||
if self._prefix_graphs_enabled:
|
||||
prefix_key = (padded_bsz, step + 1)
|
||||
if prefix_key in self._device_graphs:
|
||||
graph_key = prefix_key
|
||||
seq_len = step + 1
|
||||
pos_ids = self._bucket_pos_ids.get(graph_key)
|
||||
if pos_ids is None:
|
||||
pos_ids = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.long)
|
||||
.unsqueeze(0)
|
||||
.expand(padded_bsz, -1)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
# Use captured device graph if available, otherwise call compiled fn.
|
||||
device_graph_entry = self._device_graphs.get(graph_key)
|
||||
|
||||
# Run transformer (device graph replay or compiled forward)
|
||||
if device_graph_entry is not None:
|
||||
device_graph_entry[0].replay()
|
||||
hidden_out = device_graph_entry[1]
|
||||
else:
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :max_seq, :], full_pos_ids)
|
||||
hidden_out = model_fwd(proj_buf[:padded_bsz, :seq_len, :], pos_ids)
|
||||
|
||||
logits = lm_heads[step - 1](hidden_out[:bsz, step, :])
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@ This module provides CUDA Graph acceleration for the speech tokenizer decoder,
|
||||
reducing kernel launch overhead during inference.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Sequence
|
||||
|
||||
import torch
|
||||
from torch.cuda import CUDAGraph
|
||||
from vllm.logger import init_logger
|
||||
@@ -15,6 +20,108 @@ from vllm.platforms import current_platform
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_decode_lengths(lengths: Sequence[int], batch_size: int, max_len: int) -> tuple[int, ...]:
|
||||
if len(lengths) != batch_size:
|
||||
raise ValueError(f"Expected {batch_size} decode lengths, got {len(lengths)}")
|
||||
|
||||
normalized: list[int] = []
|
||||
for length in lengths:
|
||||
length_int = int(length)
|
||||
if length_int < 0 or length_int > max_len:
|
||||
raise ValueError(f"Invalid decode length {length_int}; expected 0 <= length <= {max_len}")
|
||||
normalized.append(length_int)
|
||||
return tuple(normalized)
|
||||
|
||||
|
||||
def _batched_chunked_decode(
|
||||
codes: torch.Tensor,
|
||||
lengths: Sequence[int],
|
||||
*,
|
||||
decode_fn: Callable[[torch.Tensor], torch.Tensor],
|
||||
total_upsample: int,
|
||||
chunk_size: int = 300,
|
||||
left_context_size: int = 25,
|
||||
max_batch_size: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""Decode a padded batch by grouping same-round chunks across requests."""
|
||||
if codes.dim() < 3:
|
||||
raise ValueError(f"Expected codes with shape [B, Q, F], got {tuple(codes.shape)}")
|
||||
if chunk_size <= 0:
|
||||
raise ValueError(f"chunk_size must be positive, got {chunk_size}")
|
||||
if left_context_size < 0:
|
||||
raise ValueError(f"left_context_size must be non-negative, got {left_context_size}")
|
||||
if max_batch_size < 0:
|
||||
raise ValueError(f"max_batch_size must be non-negative, got {max_batch_size}")
|
||||
|
||||
batch_size = int(codes.shape[0])
|
||||
max_input_len = int(codes.shape[-1])
|
||||
length_values = _normalize_decode_lengths(lengths, batch_size, max_input_len)
|
||||
max_decode_len = max(length_values, default=0)
|
||||
if max_decode_len == 0:
|
||||
return torch.empty((batch_size, 1, 0), dtype=torch.float32, device=codes.device)
|
||||
|
||||
total_upsample = int(total_upsample)
|
||||
wav_out: torch.Tensor | None = None
|
||||
num_rounds = (max_decode_len + chunk_size - 1) // chunk_size
|
||||
|
||||
for round_index in range(num_rounds):
|
||||
start_index = round_index * chunk_size
|
||||
grouped_jobs: dict[int, list[tuple[int, int, int, int, int, int]]] = {}
|
||||
for req_index, total_len in enumerate(length_values):
|
||||
if start_index >= total_len:
|
||||
continue
|
||||
end_index = min(start_index + chunk_size, total_len)
|
||||
context_size = left_context_size if start_index - left_context_size > 0 else start_index
|
||||
input_start = start_index - context_size
|
||||
input_end = end_index
|
||||
chunk_len = input_end - input_start
|
||||
grouped_jobs.setdefault(chunk_len, []).append(
|
||||
(req_index, input_start, input_end, start_index, end_index, context_size)
|
||||
)
|
||||
|
||||
for jobs in grouped_jobs.values():
|
||||
job_batches = (
|
||||
[jobs]
|
||||
if max_batch_size <= 0 or len(jobs) <= max_batch_size
|
||||
else [jobs[start : start + max_batch_size] for start in range(0, len(jobs), max_batch_size)]
|
||||
)
|
||||
for job_batch in job_batches:
|
||||
chunk_rows = [
|
||||
codes[req_index, :, input_start:input_end]
|
||||
for req_index, input_start, input_end, _, _, _ in job_batch
|
||||
]
|
||||
codes_chunk = torch.stack(
|
||||
chunk_rows,
|
||||
dim=0,
|
||||
)
|
||||
wav_chunk = decode_fn(codes_chunk)
|
||||
if wav_chunk.shape[0] != len(job_batch):
|
||||
raise ValueError(
|
||||
f"Decoder returned batch size {wav_chunk.shape[0]} for input batch size {len(job_batch)}"
|
||||
)
|
||||
if wav_out is None:
|
||||
wav_out = torch.empty(
|
||||
(batch_size, *wav_chunk.shape[1:-1], max_decode_len * total_upsample),
|
||||
dtype=wav_chunk.dtype,
|
||||
device=wav_chunk.device,
|
||||
)
|
||||
|
||||
for row, (req_index, _, _, chunk_start, chunk_end, context_size) in enumerate(job_batch):
|
||||
src_start = context_size * total_upsample
|
||||
dst_start = chunk_start * total_upsample
|
||||
dst_end = chunk_end * total_upsample
|
||||
src_end = src_start + (dst_end - dst_start)
|
||||
if src_end > wav_chunk.shape[-1]:
|
||||
raise ValueError(
|
||||
f"Decoder returned too-short chunk output: need {src_end}, got {wav_chunk.shape[-1]}"
|
||||
)
|
||||
wav_out[req_index, ..., dst_start:dst_end].copy_(wav_chunk[row, ..., src_start:src_end])
|
||||
|
||||
if wav_out is None:
|
||||
return torch.empty((batch_size, 1, 0), dtype=torch.float32, device=codes.device)
|
||||
return wav_out
|
||||
|
||||
|
||||
class CUDAGraphDecoderWrapper:
|
||||
"""
|
||||
CUDA Graph wrapper for Qwen3TTSTokenizerV2Decoder.
|
||||
@@ -34,21 +141,61 @@ class CUDAGraphDecoderWrapper:
|
||||
self,
|
||||
decoder: torch.nn.Module,
|
||||
capture_sizes: list[int] | None = None,
|
||||
capture_batch_sizes: list[int] | None = None,
|
||||
extra_capture_shapes: list[tuple[int, int]] | None = None,
|
||||
compile_shapes: list[tuple[int, int]] | None = None,
|
||||
num_quantizers: int = 8,
|
||||
enabled: bool = True,
|
||||
):
|
||||
self.decoder = decoder
|
||||
self._explicit_sizes = capture_sizes is not None
|
||||
self.capture_sizes = sorted(capture_sizes) if capture_sizes else []
|
||||
self.capture_batch_sizes = sorted(set(capture_batch_sizes or [1]))
|
||||
self.extra_capture_shapes = sorted(
|
||||
{
|
||||
(int(batch_size), int(size))
|
||||
for batch_size, size in extra_capture_shapes or []
|
||||
if int(batch_size) > 0 and int(size) > 0
|
||||
}
|
||||
)
|
||||
self.compile_shapes = sorted(
|
||||
{
|
||||
(int(batch_size), int(size))
|
||||
for batch_size, size in compile_shapes or []
|
||||
if int(batch_size) > 0 and int(size) > 0
|
||||
}
|
||||
)
|
||||
self._bucket_sizes = self.capture_sizes
|
||||
self.num_quantizers = num_quantizers
|
||||
self.enabled = enabled
|
||||
|
||||
self.graphs: dict[int, CUDAGraph] = {}
|
||||
self.static_inputs: dict[int, torch.Tensor] = {}
|
||||
self.static_outputs: dict[int, torch.Tensor] = {}
|
||||
self.graphs: dict[tuple[int, int], CUDAGraph] = {}
|
||||
self.static_inputs: dict[tuple[int, int], torch.Tensor] = {}
|
||||
self.static_outputs: dict[tuple[int, int], torch.Tensor] = {}
|
||||
self._compiled_decoder: Callable[[torch.Tensor], torch.Tensor] | None = None
|
||||
self._compiled_graphs: dict[tuple[int, int], CUDAGraph] = {}
|
||||
self._compiled_static_inputs: dict[tuple[int, int], torch.Tensor] = {}
|
||||
self._compiled_static_outputs: dict[tuple[int, int], torch.Tensor] = {}
|
||||
self._compiled_shapes: set[tuple[int, int]] = set()
|
||||
|
||||
self._warmed_up = False
|
||||
self._device = None
|
||||
self._stats_enabled = os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_CUDAGRAPH_STATS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
)
|
||||
self._stats_log_every = int(os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_CUDAGRAPH_STATS_LOG_EVERY", "0") or 0)
|
||||
self._stats_total = 0
|
||||
self._stats_hits = 0
|
||||
self._stats_compiled_hits = 0
|
||||
self._stats_fallbacks = 0
|
||||
self._stats_stream_capture_fallbacks = 0
|
||||
self._stats_requests: Counter[tuple[int, int]] = Counter()
|
||||
self._stats_hit_shapes: Counter[tuple[int, int, int]] = Counter()
|
||||
self._stats_compiled_shapes: Counter[tuple[int, int]] = Counter()
|
||||
self._stats_fallback_shapes: Counter[tuple[int, int, int]] = Counter()
|
||||
|
||||
@staticmethod
|
||||
def compute_capture_sizes(
|
||||
@@ -78,11 +225,68 @@ class CUDAGraphDecoderWrapper:
|
||||
return sorted(sizes)
|
||||
|
||||
def _get_padded_size(self, actual_size: int) -> int | None:
|
||||
for size in self.capture_sizes:
|
||||
for size in self._bucket_sizes:
|
||||
if actual_size <= size:
|
||||
return size
|
||||
return None
|
||||
|
||||
def _get_capture_shapes(self) -> list[tuple[int, int]]:
|
||||
shapes = {(batch_size, size) for batch_size in self.capture_batch_sizes for size in self.capture_sizes}
|
||||
shapes.update(self.extra_capture_shapes)
|
||||
return sorted(shapes)
|
||||
|
||||
def _record_decode_stats(
|
||||
self,
|
||||
*,
|
||||
hit: bool,
|
||||
batch_size: int,
|
||||
actual_size: int,
|
||||
padded_size: int | None,
|
||||
stream_capture: bool = False,
|
||||
compiled: bool = False,
|
||||
) -> None:
|
||||
if not self._stats_enabled:
|
||||
return
|
||||
|
||||
padded_key = int(padded_size) if padded_size is not None else -1
|
||||
self._stats_total += 1
|
||||
self._stats_requests[(batch_size, actual_size)] += 1
|
||||
if hit:
|
||||
self._stats_hits += 1
|
||||
if compiled:
|
||||
self._stats_compiled_hits += 1
|
||||
self._stats_compiled_shapes[(batch_size, actual_size)] += 1
|
||||
else:
|
||||
self._stats_hit_shapes[(batch_size, actual_size, padded_key)] += 1
|
||||
else:
|
||||
self._stats_fallbacks += 1
|
||||
self._stats_fallback_shapes[(batch_size, actual_size, padded_key)] += 1
|
||||
if stream_capture:
|
||||
self._stats_stream_capture_fallbacks += 1
|
||||
|
||||
if self._stats_log_every > 0 and self._stats_total % self._stats_log_every == 0:
|
||||
self.log_decode_stats()
|
||||
|
||||
def log_decode_stats(self) -> None:
|
||||
if not self._stats_enabled or self._stats_total == 0:
|
||||
return
|
||||
hit_rate = 100.0 * self._stats_hits / self._stats_total
|
||||
logger.info(
|
||||
"Code2Wav CUDA Graph stats: total=%d hits=%d fallbacks=%d "
|
||||
"compiled_hits=%d stream_capture_fallbacks=%d hit_rate=%.2f%% "
|
||||
"top_requests=%s top_compiled=%s top_hits=%s top_fallbacks=%s",
|
||||
self._stats_total,
|
||||
self._stats_hits,
|
||||
self._stats_fallbacks,
|
||||
self._stats_compiled_hits,
|
||||
self._stats_stream_capture_fallbacks,
|
||||
hit_rate,
|
||||
self._stats_requests.most_common(12),
|
||||
self._stats_compiled_shapes.most_common(12),
|
||||
self._stats_hit_shapes.most_common(12),
|
||||
self._stats_fallback_shapes.most_common(12),
|
||||
)
|
||||
|
||||
def warmup(
|
||||
self,
|
||||
device: torch.device,
|
||||
@@ -106,28 +310,97 @@ class CUDAGraphDecoderWrapper:
|
||||
decode_left_context=decode_left_context,
|
||||
)
|
||||
|
||||
logger.info("Starting CUDA Graph warmup for %d sizes: %s", len(self.capture_sizes), self.capture_sizes)
|
||||
self.capture_batch_sizes = [bs for bs in self.capture_batch_sizes if bs > 0]
|
||||
if not self.capture_batch_sizes:
|
||||
self.capture_batch_sizes = [1]
|
||||
|
||||
self._bucket_sizes = sorted(set(self.capture_sizes) | {size for _, size in self.extra_capture_shapes})
|
||||
capture_shapes = self._get_capture_shapes()
|
||||
|
||||
logger.info(
|
||||
"Starting CUDA Graph warmup for %d shapes: batch_sizes=%s seq_lens=%s extra_shapes=%s",
|
||||
len(capture_shapes),
|
||||
self.capture_batch_sizes,
|
||||
self.capture_sizes,
|
||||
self.extra_capture_shapes,
|
||||
)
|
||||
warmup_start_s = time.perf_counter()
|
||||
mem_before = self._get_cuda_memory_stats(device)
|
||||
|
||||
# Warmup runs to ensure CUDA memory is allocated
|
||||
for size in self.capture_sizes:
|
||||
dummy = torch.zeros(1, self.num_quantizers, size, dtype=dtype, device=device)
|
||||
for batch_size, size in capture_shapes:
|
||||
dummy = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
|
||||
with torch.no_grad():
|
||||
_ = self.decoder(dummy)
|
||||
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
for size in self.capture_sizes:
|
||||
for batch_size, size in capture_shapes:
|
||||
try:
|
||||
self._capture(size, device, dtype)
|
||||
logger.info(" Captured CUDA Graph for size=%d", size)
|
||||
self._capture(batch_size, size, device, dtype)
|
||||
logger.info(" Captured CUDA Graph for batch=%d size=%d", batch_size, size)
|
||||
except Exception:
|
||||
logger.warning(" Failed to capture graph for size=%d", size, exc_info=True)
|
||||
logger.warning(" Failed to capture graph for batch=%d size=%d", batch_size, size, exc_info=True)
|
||||
|
||||
if self.compile_shapes:
|
||||
self._warmup_compile_shapes(device, dtype)
|
||||
|
||||
self._warmed_up = True
|
||||
logger.info("CUDA Graph warmup complete: %d/%d captured", len(self.graphs), len(self.capture_sizes))
|
||||
warmup_ms = (time.perf_counter() - warmup_start_s) * 1000.0
|
||||
mem_after = self._get_cuda_memory_stats(device)
|
||||
logger.info(
|
||||
"CUDA Graph warmup complete: %d/%d captured in %.1f ms%s",
|
||||
len(self.graphs),
|
||||
len(capture_shapes),
|
||||
warmup_ms,
|
||||
self._format_cuda_memory_delta(mem_before, mem_after),
|
||||
)
|
||||
|
||||
def _capture(self, size: int, device: torch.device, dtype: torch.dtype):
|
||||
static_input = torch.zeros(1, self.num_quantizers, size, dtype=dtype, device=device)
|
||||
def _warmup_compile_shapes(self, device: torch.device, dtype: torch.dtype) -> None:
|
||||
logger.info("Starting torch.compile + CUDA Graph warmup for decoder shapes: %s", self.compile_shapes)
|
||||
compile_start_s = time.perf_counter()
|
||||
try:
|
||||
self._compiled_decoder = torch.compile(
|
||||
self.decoder.forward,
|
||||
mode="default",
|
||||
fullgraph=False,
|
||||
dynamic=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to create torch.compile decoder wrapper", exc_info=True)
|
||||
self._compiled_decoder = None
|
||||
return
|
||||
|
||||
assert self._compiled_decoder is not None
|
||||
for batch_size, size in self.compile_shapes:
|
||||
shape_start_s = time.perf_counter()
|
||||
try:
|
||||
self._capture_compiled(batch_size, size, device, dtype)
|
||||
self._compiled_shapes.add((batch_size, size))
|
||||
logger.info(
|
||||
" torch.compile + CUDA Graph ready for batch=%d size=%d in %.1f ms",
|
||||
batch_size,
|
||||
size,
|
||||
(time.perf_counter() - shape_start_s) * 1000.0,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
" Failed to capture torch.compile CUDA Graph for batch=%d size=%d; "
|
||||
"this shape will use CUDA Graph/eager fallback",
|
||||
batch_size,
|
||||
size,
|
||||
exc_info=True,
|
||||
)
|
||||
logger.info(
|
||||
"torch.compile + CUDA Graph warmup complete: %d/%d shapes ready in %.1f ms",
|
||||
len(self._compiled_shapes),
|
||||
len(self.compile_shapes),
|
||||
(time.perf_counter() - compile_start_s) * 1000.0,
|
||||
)
|
||||
|
||||
def _capture(self, batch_size: int, size: int, device: torch.device, dtype: torch.dtype):
|
||||
key = (batch_size, size)
|
||||
static_input = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
|
||||
with torch.no_grad():
|
||||
_ = self.decoder(static_input)
|
||||
torch.accelerator.synchronize(device)
|
||||
@@ -137,12 +410,64 @@ class CUDAGraphDecoderWrapper:
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_output = self.decoder(static_input)
|
||||
|
||||
self.graphs[size] = graph
|
||||
self.static_inputs[size] = static_input
|
||||
self.static_outputs[size] = static_output
|
||||
self.graphs[key] = graph
|
||||
self.static_inputs[key] = static_input
|
||||
self.static_outputs[key] = static_output
|
||||
|
||||
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
if not self.enabled or not self._warmed_up or codes.shape[0] != 1:
|
||||
def _capture_compiled(self, batch_size: int, size: int, device: torch.device, dtype: torch.dtype):
|
||||
if self._compiled_decoder is None:
|
||||
raise RuntimeError("Compiled decoder is not initialized")
|
||||
|
||||
key = (batch_size, size)
|
||||
static_input = torch.zeros(batch_size, self.num_quantizers, size, dtype=dtype, device=device)
|
||||
with torch.inference_mode():
|
||||
for _ in range(5):
|
||||
_ = self._compiled_decoder(static_input)
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
graph = CUDAGraph()
|
||||
with torch.inference_mode():
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_output = self._compiled_decoder(static_input)
|
||||
|
||||
self._compiled_graphs[key] = graph
|
||||
self._compiled_static_inputs[key] = static_input
|
||||
self._compiled_static_outputs[key] = static_output
|
||||
|
||||
@staticmethod
|
||||
def _get_cuda_memory_stats(device: torch.device) -> tuple[int, int, int] | None:
|
||||
if device.type != "cuda":
|
||||
return None
|
||||
try:
|
||||
return (
|
||||
int(torch.cuda.memory_allocated(device)),
|
||||
int(torch.cuda.memory_reserved(device)),
|
||||
int(torch.cuda.max_memory_reserved(device)),
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _format_cuda_memory_delta(
|
||||
before: tuple[int, int, int] | None,
|
||||
after: tuple[int, int, int] | None,
|
||||
) -> str:
|
||||
if before is None or after is None:
|
||||
return ""
|
||||
|
||||
def gib(value: int) -> float:
|
||||
return value / 1024**3
|
||||
|
||||
alloc_before, reserved_before, max_reserved_before = before
|
||||
alloc_after, reserved_after, max_reserved_after = after
|
||||
return (
|
||||
f" (cuda_mem allocated {gib(alloc_before):.2f}->{gib(alloc_after):.2f} GiB, "
|
||||
f"reserved {gib(reserved_before):.2f}->{gib(reserved_after):.2f} GiB, "
|
||||
f"max_reserved {gib(max_reserved_before):.2f}->{gib(max_reserved_after):.2f} GiB)"
|
||||
)
|
||||
|
||||
def _decode(self, codes: torch.Tensor, *, clone_graph_output: bool) -> torch.Tensor:
|
||||
if not self.enabled or not self._warmed_up:
|
||||
return self.decoder(codes)
|
||||
|
||||
# Inner CUDA graph replay is illegal while an outer stream capture is
|
||||
@@ -152,20 +477,76 @@ class CUDAGraphDecoderWrapper:
|
||||
# outside the startup capture window, so normal inference still hits
|
||||
# the graph fast path.
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
self._record_decode_stats(
|
||||
hit=False,
|
||||
batch_size=int(codes.shape[0]),
|
||||
actual_size=int(codes.shape[-1]),
|
||||
padded_size=None,
|
||||
stream_capture=True,
|
||||
)
|
||||
return self.decoder(codes)
|
||||
|
||||
actual_size = codes.shape[-1]
|
||||
batch_size = int(codes.shape[0])
|
||||
actual_size = int(codes.shape[-1])
|
||||
padded_size = self._get_padded_size(actual_size)
|
||||
compile_key = (batch_size, actual_size)
|
||||
if compile_key not in self._compiled_shapes and padded_size is not None:
|
||||
compile_key = (batch_size, padded_size)
|
||||
if compile_key in self._compiled_shapes:
|
||||
compiled_size = compile_key[1]
|
||||
self._record_decode_stats(
|
||||
hit=True,
|
||||
batch_size=batch_size,
|
||||
actual_size=actual_size,
|
||||
padded_size=compiled_size,
|
||||
compiled=True,
|
||||
)
|
||||
static_input = self._compiled_static_inputs[compile_key]
|
||||
if actual_size == compiled_size:
|
||||
static_input.copy_(codes)
|
||||
else:
|
||||
static_input.zero_()
|
||||
static_input[:, :, :actual_size] = codes
|
||||
self._compiled_graphs[compile_key].replay()
|
||||
actual_out_len = actual_size * self.decoder.total_upsample
|
||||
output = self._compiled_static_outputs[compile_key][..., :actual_out_len]
|
||||
if clone_graph_output:
|
||||
return output.clone()
|
||||
return output
|
||||
|
||||
if padded_size is None or padded_size not in self.graphs:
|
||||
graph_key = (batch_size, padded_size) if padded_size is not None else None
|
||||
|
||||
if graph_key is None or graph_key not in self.graphs:
|
||||
self._record_decode_stats(
|
||||
hit=False,
|
||||
batch_size=batch_size,
|
||||
actual_size=actual_size,
|
||||
padded_size=padded_size,
|
||||
)
|
||||
return self.decoder(codes)
|
||||
|
||||
self.static_inputs[padded_size].zero_()
|
||||
self.static_inputs[padded_size][:, :, :actual_size] = codes
|
||||
self.graphs[padded_size].replay()
|
||||
self._record_decode_stats(
|
||||
hit=True,
|
||||
batch_size=batch_size,
|
||||
actual_size=actual_size,
|
||||
padded_size=padded_size,
|
||||
)
|
||||
static_input = self.static_inputs[graph_key]
|
||||
if actual_size == padded_size:
|
||||
static_input.copy_(codes)
|
||||
else:
|
||||
static_input.zero_()
|
||||
static_input[:, :, :actual_size] = codes
|
||||
self.graphs[graph_key].replay()
|
||||
|
||||
actual_out_len = actual_size * self.decoder.total_upsample
|
||||
return self.static_outputs[padded_size][..., :actual_out_len].clone()
|
||||
output = self.static_outputs[graph_key][..., :actual_out_len]
|
||||
if clone_graph_output:
|
||||
return output.clone()
|
||||
return output
|
||||
|
||||
def decode(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
return self._decode(codes, clone_graph_output=True)
|
||||
|
||||
def chunked_decode_with_cudagraph(
|
||||
self,
|
||||
@@ -173,19 +554,48 @@ class CUDAGraphDecoderWrapper:
|
||||
chunk_size: int = 300,
|
||||
left_context_size: int = 25,
|
||||
) -> torch.Tensor:
|
||||
wavs = []
|
||||
start_index = 0
|
||||
total_len = codes.shape[-1]
|
||||
total_upsample = self.decoder.total_upsample
|
||||
wav_out = None
|
||||
|
||||
while start_index < total_len:
|
||||
end_index = min(start_index + chunk_size, total_len)
|
||||
context_size = left_context_size if start_index - left_context_size > 0 else start_index
|
||||
|
||||
codes_chunk = codes[..., start_index - context_size : end_index]
|
||||
wav_chunk = self.decode(codes_chunk)
|
||||
wav_chunk = self._decode(codes_chunk, clone_graph_output=False)
|
||||
|
||||
wavs.append(wav_chunk[..., context_size * total_upsample :])
|
||||
if wav_out is None:
|
||||
wav_out = torch.empty(
|
||||
(*wav_chunk.shape[:-1], total_len * total_upsample),
|
||||
dtype=wav_chunk.dtype,
|
||||
device=wav_chunk.device,
|
||||
)
|
||||
src_start = context_size * total_upsample
|
||||
dst_start = start_index * total_upsample
|
||||
dst_end = end_index * total_upsample
|
||||
wav_out[..., dst_start:dst_end].copy_(wav_chunk[..., src_start:])
|
||||
start_index = end_index
|
||||
|
||||
return torch.cat(wavs, dim=-1)
|
||||
if wav_out is None:
|
||||
return self.decoder(codes)
|
||||
return wav_out
|
||||
|
||||
def batched_chunked_decode_with_cudagraph(
|
||||
self,
|
||||
codes: torch.Tensor,
|
||||
lengths: Sequence[int],
|
||||
chunk_size: int = 300,
|
||||
left_context_size: int = 25,
|
||||
max_batch_size: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return _batched_chunked_decode(
|
||||
codes,
|
||||
lengths,
|
||||
decode_fn=lambda codes_chunk: self._decode(codes_chunk, clone_graph_output=False),
|
||||
total_upsample=self.decoder.total_upsample,
|
||||
chunk_size=chunk_size,
|
||||
left_context_size=left_context_size,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
@@ -43,7 +45,25 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
|
||||
self._decode_chunk_frames = 300
|
||||
self._decode_left_context_frames = 25
|
||||
self._decode_batch_bucket_frames: list[int] = []
|
||||
self._decode_batch_max_size = 0
|
||||
self._decode_variable_chunk_batch_min_frames = self._decode_chunk_frames + self._decode_left_context_frames + 1
|
||||
self._logged_codec_stats = False
|
||||
self._logged_malformed_codec_lengths: set[tuple[int, int]] = set()
|
||||
self._batch_stats_enabled = os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
)
|
||||
self._batch_stats_log_every = int(os.environ.get("VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS_LOG_EVERY", "0") or 0)
|
||||
self._batch_stats_forwards = 0
|
||||
self._batch_stats_groups = 0
|
||||
self._batch_stats_requests = 0
|
||||
self._batch_stats_padded_frames = 0
|
||||
self._batch_stats_decoded_frames = 0
|
||||
self._batch_stats_actual_frames: Counter[int] = Counter()
|
||||
self._batch_stats_bucket_groups: Counter[tuple[int, int]] = Counter()
|
||||
|
||||
# Construct decoder from config so it is visible to vLLM's
|
||||
# memory profiler at startup. Weights are loaded later in
|
||||
@@ -91,6 +111,50 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
|
||||
return [ids]
|
||||
|
||||
def _get_decode_batch_bucket_frames(self, actual_frames: int) -> int:
|
||||
for bucket_frames in self._decode_batch_bucket_frames:
|
||||
if actual_frames <= bucket_frames:
|
||||
return bucket_frames
|
||||
return actual_frames
|
||||
|
||||
def _record_decode_batch_stats(
|
||||
self,
|
||||
*,
|
||||
group_size: int,
|
||||
bucket_frames: int,
|
||||
actual_frames: list[int],
|
||||
) -> None:
|
||||
if not self._batch_stats_enabled:
|
||||
return
|
||||
|
||||
self._batch_stats_groups += 1
|
||||
self._batch_stats_requests += group_size
|
||||
self._batch_stats_decoded_frames += group_size * bucket_frames
|
||||
self._batch_stats_padded_frames += sum(bucket_frames - frames for frames in actual_frames)
|
||||
self._batch_stats_actual_frames.update(actual_frames)
|
||||
self._batch_stats_bucket_groups[(group_size, bucket_frames)] += 1
|
||||
|
||||
def log_decode_batch_stats(self) -> None:
|
||||
if not self._batch_stats_enabled or self._batch_stats_requests == 0:
|
||||
return
|
||||
|
||||
avg_group_size = self._batch_stats_requests / max(1, self._batch_stats_groups)
|
||||
pad_ratio = self._batch_stats_padded_frames / max(1, self._batch_stats_decoded_frames)
|
||||
logger.info(
|
||||
"Code2Wav batch stats: forwards=%d groups=%d requests=%d "
|
||||
"avg_group_size=%.2f padded_frames=%d decoded_frames=%d pad_ratio=%.2f%% "
|
||||
"top_actual_frames=%s top_bucket_groups=%s",
|
||||
self._batch_stats_forwards,
|
||||
self._batch_stats_groups,
|
||||
self._batch_stats_requests,
|
||||
avg_group_size,
|
||||
self._batch_stats_padded_frames,
|
||||
self._batch_stats_decoded_frames,
|
||||
100.0 * pad_ratio,
|
||||
self._batch_stats_actual_frames.most_common(12),
|
||||
self._batch_stats_bucket_groups.most_common(12),
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
@@ -111,6 +175,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
Length management is done here instead of relying on HF's padding=-1
|
||||
sentinel logic.
|
||||
"""
|
||||
self._batch_stats_forwards += 1
|
||||
decoder = self.decoder
|
||||
q = int(self._num_quantizers)
|
||||
upsample = int(self._total_upsample)
|
||||
@@ -147,11 +212,15 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
n = flat.numel()
|
||||
if n == 0 or n % q != 0:
|
||||
if n > 0:
|
||||
logger.warning(
|
||||
"Code2Wav input_ids length %d not divisible by num_quantizers %d; skipping malformed request.",
|
||||
n,
|
||||
q,
|
||||
)
|
||||
key = (int(n), q)
|
||||
if key not in self._logged_malformed_codec_lengths:
|
||||
self._logged_malformed_codec_lengths.add(key)
|
||||
logger.warning(
|
||||
"Code2Wav input_ids length %d not divisible by num_quantizers %d; "
|
||||
"skipping malformed request and suppressing repeats for this length.",
|
||||
n,
|
||||
q,
|
||||
)
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
frames = n // q
|
||||
@@ -187,22 +256,95 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Decode directly via decoder.chunked_decode(), staying entirely on GPU.
|
||||
# Each request decoded individually with CUDA graph replay at bs=1.
|
||||
wav_tensors: list[torch.Tensor] = []
|
||||
for codes_qf in valid_codes_qf:
|
||||
codes_bqf = codes_qf.unsqueeze(0) # [1, Q, F]
|
||||
try:
|
||||
wav = decoder.chunked_decode(
|
||||
codes_bqf,
|
||||
chunk_size=self._decode_chunk_frames,
|
||||
left_context_size=self._decode_left_context_frames,
|
||||
) # [1, 1, wav_len]
|
||||
except TypeError:
|
||||
# Unit-test fakes and older decoder shims may not accept the
|
||||
# explicit chunk kwargs; production Qwen3-TTS decoders do.
|
||||
wav = decoder.chunked_decode(codes_bqf) # [1, 1, wav_len]
|
||||
wav_tensors.append(wav.squeeze(0).squeeze(0)) # [wav_len]
|
||||
wav_tensors: list[torch.Tensor | None] = [None] * len(valid_codes_qf)
|
||||
|
||||
def _decode_group_chunks(group_chunks: list[list[tuple[int, torch.Tensor]]]) -> None:
|
||||
for group_chunk in group_chunks:
|
||||
actual_frames = [int(codes_qf.shape[1]) for _, codes_qf in group_chunk]
|
||||
target_frames = max(actual_frames)
|
||||
is_equal_length_batch = all(frames == target_frames for frames in actual_frames)
|
||||
use_variable_length_batch = (
|
||||
len(group_chunk) > 1
|
||||
and not is_equal_length_batch
|
||||
and target_frames >= self._decode_variable_chunk_batch_min_frames
|
||||
and hasattr(decoder, "batched_chunked_decode")
|
||||
)
|
||||
if len(group_chunk) == 1:
|
||||
codes_bqf = group_chunk[0][1].unsqueeze(0)
|
||||
elif is_equal_length_batch:
|
||||
codes_bqf = torch.stack([codes_qf for _, codes_qf in group_chunk], dim=0)
|
||||
else:
|
||||
first = group_chunk[0][1]
|
||||
codes_bqf = first.new_zeros((len(group_chunk), q, target_frames))
|
||||
for row, (_, codes_qf) in enumerate(group_chunk):
|
||||
codes_bqf[row, :, : codes_qf.shape[1]] = codes_qf
|
||||
self._record_decode_batch_stats(
|
||||
group_size=len(group_chunk),
|
||||
bucket_frames=target_frames,
|
||||
actual_frames=actual_frames,
|
||||
)
|
||||
try:
|
||||
if use_variable_length_batch:
|
||||
wav_batch = decoder.batched_chunked_decode(
|
||||
codes_bqf,
|
||||
actual_frames,
|
||||
chunk_size=self._decode_chunk_frames,
|
||||
left_context_size=self._decode_left_context_frames,
|
||||
max_batch_size=self._decode_batch_max_size,
|
||||
) # [B, 1, wav_len]
|
||||
else:
|
||||
wav_batch = decoder.chunked_decode(
|
||||
codes_bqf,
|
||||
chunk_size=self._decode_chunk_frames,
|
||||
left_context_size=self._decode_left_context_frames,
|
||||
) # [B, 1, wav_len]
|
||||
except TypeError:
|
||||
# Unit-test fakes and older decoder shims may not accept the
|
||||
# explicit chunk kwargs; production Qwen3-TTS decoders do.
|
||||
wav_batch = decoder.chunked_decode(codes_bqf) # [B, 1, wav_len]
|
||||
|
||||
if wav_batch.dim() == 3 and wav_batch.shape[1] == 1:
|
||||
wav_rows = wav_batch[:, 0, :]
|
||||
elif wav_batch.dim() == 2:
|
||||
wav_rows = wav_batch
|
||||
else:
|
||||
raise ValueError(
|
||||
"Code2Wav decoder returned unexpected shape "
|
||||
f"{tuple(wav_batch.shape)} for batch size {len(group_chunk)}"
|
||||
)
|
||||
if wav_rows.shape[0] != len(group_chunk):
|
||||
raise ValueError(
|
||||
f"Code2Wav decoder returned batch size {wav_rows.shape[0]} "
|
||||
f"for input batch size {len(group_chunk)}"
|
||||
)
|
||||
for row, (j, _) in enumerate(group_chunk):
|
||||
wav_tensors[j] = wav_rows[row]
|
||||
|
||||
# Group by configured frame buckets instead of only exact lengths.
|
||||
# For ordinary async streaming windows this is the real batching
|
||||
# opportunity; decoder-internal variable chunk batching is gated to
|
||||
# longer inputs where repeated full chunks can amortize its overhead.
|
||||
grouped_codes: dict[int, list[tuple[int, torch.Tensor]]] = {}
|
||||
for j, codes_qf in enumerate(valid_codes_qf):
|
||||
frames = int(codes_qf.shape[1])
|
||||
grouped_codes.setdefault(self._get_decode_batch_bucket_frames(frames), []).append((j, codes_qf))
|
||||
|
||||
for _bucket_frames, group in grouped_codes.items():
|
||||
if self._decode_batch_max_size > 0 and len(group) > self._decode_batch_max_size:
|
||||
# Keep each decoder call inside the configured CUDA graph batch
|
||||
# envelope. Sorting by length lowers right-padding within each
|
||||
# split while outputs are restored by original request index.
|
||||
group = sorted(group, key=lambda item: int(item[1].shape[1]))
|
||||
group_chunks = [
|
||||
group[start : start + self._decode_batch_max_size]
|
||||
for start in range(0, len(group), self._decode_batch_max_size)
|
||||
]
|
||||
else:
|
||||
group_chunks = [group]
|
||||
_decode_group_chunks(group_chunks)
|
||||
|
||||
if self._batch_stats_log_every > 0 and self._batch_stats_forwards % self._batch_stats_log_every == 0:
|
||||
self.log_decode_batch_stats()
|
||||
|
||||
audios: list[torch.Tensor] = [empty] * num_req
|
||||
srs = [sr_tensor] * num_req
|
||||
@@ -210,6 +352,7 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
for j, idx in enumerate(valid_indices):
|
||||
ctx_frames, actual_frames = parsed[idx]
|
||||
wav = wav_tensors[j]
|
||||
assert wav is not None
|
||||
# Slice on exact codec-frame boundaries instead of proportionally.
|
||||
start = max(0, ctx_frames * upsample)
|
||||
end = max(start, actual_frames * upsample)
|
||||
@@ -302,6 +445,80 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
|
||||
def _get_bool_config(name: str, default: bool) -> bool:
|
||||
value = extra_cfg.get(name, default)
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in ("1", "true", "yes", "on"):
|
||||
return True
|
||||
if lowered in ("0", "false", "no", "off"):
|
||||
return False
|
||||
if isinstance(value, int):
|
||||
return bool(value)
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
|
||||
|
||||
def _get_int_list_config(name: str) -> list[int] | None:
|
||||
value = extra_cfg.get(name)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
raw_values = [item.strip() for item in value.split(",") if item.strip()]
|
||||
elif isinstance(value, int):
|
||||
raw_values = [value]
|
||||
else:
|
||||
try:
|
||||
raw_values = list(value)
|
||||
except TypeError as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
values: set[int] = set()
|
||||
for item in raw_values:
|
||||
try:
|
||||
parsed = int(item)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
if parsed > 0:
|
||||
values.add(parsed)
|
||||
return sorted(values)
|
||||
|
||||
def _get_int_pair_list_config(name: str) -> list[tuple[int, int]] | None:
|
||||
value = extra_cfg.get(name)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
raw_values = [item.strip() for item in value.split(",") if item.strip()]
|
||||
else:
|
||||
try:
|
||||
raw_values = list(value)
|
||||
except TypeError as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
|
||||
pairs: set[tuple[int, int]] = set()
|
||||
for item in raw_values:
|
||||
if isinstance(item, str):
|
||||
if ":" not in item:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
|
||||
left, right = item.split(":", 1)
|
||||
raw_pair = (left.strip(), right.strip())
|
||||
else:
|
||||
try:
|
||||
raw_pair = tuple(item)
|
||||
except TypeError as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
if len(raw_pair) != 2:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}")
|
||||
try:
|
||||
batch_size = int(raw_pair[0])
|
||||
seq_len = int(raw_pair[1])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config {name}={value!r}") from exc
|
||||
if batch_size > 0 and seq_len > 0:
|
||||
pairs.add((batch_size, seq_len))
|
||||
return sorted(pairs)
|
||||
|
||||
if isinstance(extra_cfg, dict):
|
||||
codec_chunk_frames = int(extra_cfg.get("codec_chunk_frames") or 0)
|
||||
codec_left_context_frames = int(extra_cfg.get("codec_left_context_frames") or 0)
|
||||
@@ -318,6 +535,49 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
)
|
||||
self._decode_chunk_frames = decode_chunk_frames
|
||||
self._decode_left_context_frames = decode_left_context_frames
|
||||
decode_cudagraph_capture_sizes = _get_int_list_config("decode_cudagraph_capture_sizes")
|
||||
decode_cudagraph_batch_sizes = _get_int_list_config("decode_cudagraph_batch_sizes")
|
||||
decode_cudagraph_extra_capture_shapes = _get_int_pair_list_config("decode_cudagraph_extra_capture_shapes")
|
||||
decode_compile_shapes = _get_int_pair_list_config("decode_compile_shapes")
|
||||
decode_batch_bucket_frames = _get_int_list_config("decode_batch_bucket_frames")
|
||||
if decode_batch_bucket_frames is not None:
|
||||
self._decode_batch_bucket_frames = decode_batch_bucket_frames
|
||||
decode_batch_max_size = _get_int_config("decode_batch_max_size", self._decode_batch_max_size)
|
||||
if decode_batch_max_size < 0:
|
||||
raise ValueError(f"Invalid Qwen3-TTS Code2Wav config decode_batch_max_size={decode_batch_max_size}")
|
||||
self._decode_batch_max_size = decode_batch_max_size
|
||||
decode_variable_chunk_batch_min_frames = _get_int_config(
|
||||
"decode_variable_chunk_batch_min_frames",
|
||||
self._decode_variable_chunk_batch_min_frames,
|
||||
)
|
||||
if decode_variable_chunk_batch_min_frames < 0:
|
||||
raise ValueError(
|
||||
"Invalid Qwen3-TTS Code2Wav config "
|
||||
f"decode_variable_chunk_batch_min_frames={decode_variable_chunk_batch_min_frames}"
|
||||
)
|
||||
self._decode_variable_chunk_batch_min_frames = decode_variable_chunk_batch_min_frames
|
||||
decode_enable_tf32 = _get_bool_config("decode_enable_tf32", False)
|
||||
else:
|
||||
decode_cudagraph_capture_sizes = None
|
||||
decode_cudagraph_batch_sizes = None
|
||||
decode_cudagraph_extra_capture_shapes = None
|
||||
decode_compile_shapes = None
|
||||
decode_enable_tf32 = False
|
||||
|
||||
if decode_enable_tf32 and device.type == "cuda":
|
||||
# PyTorch exposes TF32 controls as process-wide CUDA backend
|
||||
# switches. This opt-in is intended for deployments where
|
||||
# Code2Wav runs in its own Stage1 worker process.
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
torch.set_float32_matmul_precision("high")
|
||||
logger.info(
|
||||
"Qwen3-TTS Code2Wav TF32 enabled process-wide: "
|
||||
"matmul.allow_tf32=%s cudnn.allow_tf32=%s float32_matmul_precision=%s",
|
||||
torch.backends.cuda.matmul.allow_tf32,
|
||||
torch.backends.cudnn.allow_tf32,
|
||||
torch.get_float32_matmul_precision(),
|
||||
)
|
||||
|
||||
if hasattr(self.decoder, "enable_cudagraph") and device.type == "cuda":
|
||||
try:
|
||||
@@ -339,6 +599,10 @@ class Qwen3TTSCode2Wav(nn.Module):
|
||||
)
|
||||
|
||||
self.decoder.enable_cudagraph(
|
||||
capture_sizes=decode_cudagraph_capture_sizes,
|
||||
capture_batch_sizes=decode_cudagraph_batch_sizes,
|
||||
extra_capture_shapes=decode_cudagraph_extra_capture_shapes,
|
||||
compile_shapes=decode_compile_shapes,
|
||||
device=device,
|
||||
codec_chunk_frames=codec_chunk_frames,
|
||||
codec_left_context_frames=codec_left_context_frames,
|
||||
|
||||
@@ -37,6 +37,12 @@ from .qwen3_tts_tokenizer import Qwen3TTSTokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_TRAILING_TEXT_COMPACT_MIN_FRAMES = 64
|
||||
_PRECOMPUTED_REF_CODE_KEY = "precomputed_ref"
|
||||
_NORMALIZED_REF_AUDIO_KEY = "_qwen3_tts_normalized_ref_audio"
|
||||
_PRECOMPUTED_TEXT_IDS_KEY = "_qwen3_tts_text_ids"
|
||||
_PRECOMPUTED_REF_IDS_KEY = "_qwen3_tts_ref_ids"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Components ported from the HuggingFace Qwen3-TTS reference implementation.
|
||||
@@ -259,6 +265,8 @@ def mel_spectrogram(
|
||||
fmin: int,
|
||||
fmax: int | None = None,
|
||||
center: bool = False,
|
||||
mel_basis: torch.Tensor | None = None,
|
||||
hann_window: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Calculate mel spectrogram of an input signal using torchaudio mel filterbank and torch STFT."""
|
||||
if torch.min(y) < -1.0:
|
||||
@@ -266,14 +274,20 @@ def mel_spectrogram(
|
||||
if torch.max(y) > 1.0:
|
||||
logger.warning("Max value of input waveform signal is %s", torch.max(y))
|
||||
device = y.device
|
||||
mel_basis = mel_filter_bank(
|
||||
sr=sampling_rate,
|
||||
n_fft=n_fft,
|
||||
n_mels=num_mels,
|
||||
fmin=fmin,
|
||||
fmax=fmax,
|
||||
).to(device)
|
||||
hann_window = torch.hann_window(win_size).to(device)
|
||||
if mel_basis is None:
|
||||
mel_basis = mel_filter_bank(
|
||||
sr=sampling_rate,
|
||||
n_fft=n_fft,
|
||||
n_mels=num_mels,
|
||||
fmin=fmin,
|
||||
fmax=fmax,
|
||||
).to(device)
|
||||
elif mel_basis.device != device:
|
||||
mel_basis = mel_basis.to(device)
|
||||
if hann_window is None:
|
||||
hann_window = torch.hann_window(win_size, device=device)
|
||||
elif hann_window.device != device:
|
||||
hann_window = hann_window.to(device)
|
||||
padding = (n_fft - hop_size) // 2
|
||||
y = torch.nn.functional.pad(y.unsqueeze(1), (padding, padding), mode="reflect").squeeze(1)
|
||||
spec = torch.stft(
|
||||
@@ -560,6 +574,14 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
span_len = int(input_ids.shape[0])
|
||||
if span_len <= 0:
|
||||
return input_ids, input_embeds if input_embeds is not None else self.embed_input_ids(input_ids), {}
|
||||
is_prefill_raw = info_dict.get("_omni_is_prefill")
|
||||
if isinstance(is_prefill_raw, bool):
|
||||
is_prefill = is_prefill_raw
|
||||
else:
|
||||
try:
|
||||
is_prefill = int(info_dict["_omni_num_computed_tokens"]) < int(info_dict["_omni_prompt_len"])
|
||||
except Exception:
|
||||
is_prefill = span_len > 1
|
||||
|
||||
text_list = info_dict.get("text")
|
||||
if not isinstance(text_list, list) or not text_list or not text_list[0]:
|
||||
@@ -576,7 +598,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
else:
|
||||
codec_streaming = task_type == "Base"
|
||||
|
||||
if span_len > 1:
|
||||
if is_prefill:
|
||||
# Prefill (prompt embeddings)
|
||||
prompt_embeds_cpu = embed.get("prefill")
|
||||
tts_pad_embed_cpu = embed.get("tts_pad")
|
||||
@@ -600,7 +622,11 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
"tts_pad": tts_pad_embed.detach(),
|
||||
},
|
||||
"hidden_states": {"trailing_text": tailing_text_hidden.detach()},
|
||||
"meta": {"talker_prefill_offset": 0, "codec_streaming": codec_streaming},
|
||||
"meta": {
|
||||
"talker_prefill_offset": 0,
|
||||
"talker_text_offset": 0,
|
||||
"codec_streaming": codec_streaming,
|
||||
},
|
||||
}
|
||||
if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0:
|
||||
info_update.setdefault("codes", {})["ref"] = ref_code.detach().to("cpu").contiguous()
|
||||
@@ -634,7 +660,10 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
take = torch.cat([take, pad_rows], dim=0)
|
||||
prompt_embeds = take.to(device=input_ids.device, dtype=torch.bfloat16)
|
||||
info_update = {
|
||||
"meta": {"talker_prefill_offset": int(offset + span_len), "codec_streaming": codec_streaming}
|
||||
"meta": {
|
||||
"talker_prefill_offset": int(offset + span_len),
|
||||
"codec_streaming": codec_streaming,
|
||||
}
|
||||
}
|
||||
|
||||
# When inputs_embeds is set, token ids are ignored by the model but must stay in-vocab for vLLM bookkeeping.
|
||||
@@ -658,12 +687,37 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
tts_pad_embed = tts_pad_embed_buf.to(device=input_ids.device, dtype=torch.bfloat16).reshape(1, -1)
|
||||
|
||||
tail = hs.get("trailing_text")
|
||||
if isinstance(tail, torch.Tensor) and tail.ndim == 2 and tail.shape[0] > 0:
|
||||
text_step = tail[:1].to(device=input_ids.device, dtype=torch.bfloat16).reshape(1, -1)
|
||||
new_tail = tail[1:] if tail.shape[0] > 1 else tail[:0]
|
||||
text_offset = max(0, int(meta.get("talker_text_offset", 0) or 0))
|
||||
trailing_text_update = None
|
||||
if isinstance(tail, torch.Tensor) and tail.ndim == 2:
|
||||
tail_len = int(tail.shape[0])
|
||||
if text_offset < tail_len:
|
||||
text_step = (
|
||||
tail[text_offset : text_offset + 1]
|
||||
.to(
|
||||
device=input_ids.device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
.reshape(1, -1)
|
||||
)
|
||||
next_text_offset = text_offset + 1
|
||||
should_compact_tail = next_text_offset >= tail_len or (
|
||||
next_text_offset >= _TRAILING_TEXT_COMPACT_MIN_FRAMES and next_text_offset * 2 >= tail_len
|
||||
)
|
||||
if should_compact_tail:
|
||||
if next_text_offset >= tail_len:
|
||||
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
|
||||
else:
|
||||
trailing_text_update = tail[next_text_offset:].contiguous()
|
||||
next_text_offset = 0
|
||||
else:
|
||||
text_step = tts_pad_embed
|
||||
next_text_offset = 0
|
||||
if tail.numel() > 0:
|
||||
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
|
||||
else:
|
||||
text_step = tts_pad_embed
|
||||
new_tail = tail if isinstance(tail, torch.Tensor) else torch.empty((0, tts_pad_embed.shape[-1]))
|
||||
next_text_offset = text_offset
|
||||
|
||||
last_hidden = hs.get("last")
|
||||
if not isinstance(last_hidden, torch.Tensor):
|
||||
@@ -677,12 +731,127 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
inputs_embeds_out = last_id_hidden.reshape(1, -1)
|
||||
|
||||
info_update = {
|
||||
"hidden_states": {"trailing_text": new_tail},
|
||||
"mtp_inputs": (past_hidden, text_step),
|
||||
"meta": {"codec_streaming": codec_streaming},
|
||||
"meta": {
|
||||
"talker_text_offset": int(next_text_offset),
|
||||
"codec_streaming": codec_streaming,
|
||||
},
|
||||
}
|
||||
if trailing_text_update is not None:
|
||||
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
|
||||
return input_ids, inputs_embeds_out, info_update
|
||||
|
||||
def preprocess_decode_batch(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
req_infos: list[dict[str, Any]],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, list[dict[str, Any]]]:
|
||||
"""Batch the decode-only preprocess path for Qwen3-TTS.
|
||||
|
||||
This mirrors the scalar decode branch in ``preprocess()``, but performs
|
||||
the token embedding lookup once for the whole decode batch.
|
||||
"""
|
||||
input_ids_flat = input_ids.reshape(-1)
|
||||
if int(input_ids_flat.numel()) != len(req_infos):
|
||||
raise ValueError(
|
||||
f"preprocess_decode_batch expected {len(req_infos)} input ids, got {int(input_ids_flat.numel())}"
|
||||
)
|
||||
|
||||
device = input_ids_flat.device
|
||||
dtype = torch.bfloat16
|
||||
past_hidden_list: list[torch.Tensor] = []
|
||||
text_step_list: list[torch.Tensor] = []
|
||||
updates: list[dict[str, Any]] = []
|
||||
|
||||
for info_dict in req_infos:
|
||||
additional_information = info_dict.get("additional_information")
|
||||
if isinstance(additional_information, dict):
|
||||
merged: dict[str, Any] = {k: v for k, v in info_dict.items() if k != "additional_information"}
|
||||
for k, v in additional_information.items():
|
||||
merged.setdefault(k, v)
|
||||
info_dict = merged
|
||||
|
||||
payload: OmniPayload = info_dict
|
||||
embed = payload.get("embed", {})
|
||||
hs = payload.get("hidden_states", {})
|
||||
meta = payload.get("meta", {})
|
||||
|
||||
text_list = info_dict.get("text")
|
||||
if not isinstance(text_list, list) or not text_list or not text_list[0]:
|
||||
raise ValueError("Missing additional_information.text for Qwen3-TTS AR talker.")
|
||||
|
||||
task_type = (info_dict.get("task_type") or ["CustomVoice"])[0]
|
||||
codec_streaming_val = meta.get("codec_streaming")
|
||||
if isinstance(codec_streaming_val, list):
|
||||
codec_streaming_raw = codec_streaming_val[0] if codec_streaming_val else None
|
||||
else:
|
||||
codec_streaming_raw = codec_streaming_val
|
||||
if isinstance(codec_streaming_raw, bool):
|
||||
codec_streaming = codec_streaming_raw
|
||||
else:
|
||||
codec_streaming = task_type == "Base"
|
||||
|
||||
tts_pad_embed_buf = embed.get("tts_pad")
|
||||
if not isinstance(tts_pad_embed_buf, torch.Tensor):
|
||||
raise RuntimeError("Missing `tts_pad_embed` in additional_information; prefill must run first.")
|
||||
tts_pad_embed = tts_pad_embed_buf.to(device=device, dtype=dtype).reshape(1, -1)
|
||||
|
||||
tail = hs.get("trailing_text")
|
||||
text_offset = max(0, int(meta.get("talker_text_offset", 0) or 0))
|
||||
trailing_text_update = None
|
||||
if isinstance(tail, torch.Tensor) and tail.ndim == 2:
|
||||
tail_len = int(tail.shape[0])
|
||||
if text_offset < tail_len:
|
||||
text_step = tail[text_offset : text_offset + 1].to(device=device, dtype=dtype).reshape(1, -1)
|
||||
next_text_offset = text_offset + 1
|
||||
should_compact_tail = next_text_offset >= tail_len or (
|
||||
next_text_offset >= _TRAILING_TEXT_COMPACT_MIN_FRAMES and next_text_offset * 2 >= tail_len
|
||||
)
|
||||
if should_compact_tail:
|
||||
if next_text_offset >= tail_len:
|
||||
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
|
||||
else:
|
||||
trailing_text_update = tail[next_text_offset:].contiguous()
|
||||
next_text_offset = 0
|
||||
else:
|
||||
text_step = tts_pad_embed
|
||||
next_text_offset = 0
|
||||
if tail.numel() > 0:
|
||||
trailing_text_update = torch.empty((0, tail.shape[1]), device=tail.device, dtype=tail.dtype)
|
||||
else:
|
||||
text_step = tts_pad_embed
|
||||
next_text_offset = text_offset
|
||||
|
||||
last_hidden = hs.get("last")
|
||||
if not isinstance(last_hidden, torch.Tensor):
|
||||
raise RuntimeError("Missing hidden_states['last'] in additional_information; postprocess must run.")
|
||||
past_hidden_list.append(last_hidden.to(device=device, dtype=dtype).reshape(1, -1))
|
||||
text_step_list.append(text_step)
|
||||
|
||||
info_update: dict[str, Any] = {
|
||||
"meta": {
|
||||
"talker_text_offset": int(next_text_offset),
|
||||
"codec_streaming": codec_streaming,
|
||||
},
|
||||
}
|
||||
if trailing_text_update is not None:
|
||||
info_update["hidden_states"] = {"trailing_text": trailing_text_update.detach()}
|
||||
updates.append(info_update)
|
||||
|
||||
inputs_embeds_out = self.embed_input_ids(input_ids_flat.reshape(-1, 1).to(torch.long)).to(
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
inputs_embeds_out = inputs_embeds_out.reshape(len(req_infos), -1)
|
||||
return (
|
||||
input_ids_flat,
|
||||
inputs_embeds_out,
|
||||
torch.cat(past_hidden_list, dim=0),
|
||||
torch.cat(text_step_list, dim=0),
|
||||
updates,
|
||||
)
|
||||
|
||||
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
|
||||
# Keep the last token hidden for the next decode step's code predictor.
|
||||
# Stays on GPU - gpu_resident_buffer_keys avoids the CPU round-trip.
|
||||
@@ -1121,9 +1290,21 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
wav = resampler.resample(wav.astype(np.float32), orig_sr=int(sr))
|
||||
sr = target_sr
|
||||
|
||||
# Follow official implementation: mel_spectrogram expects 24kHz.
|
||||
# Follow official implementation: mel_spectrogram expects 24kHz. Move
|
||||
# the waveform first so STFT/mel computation stays on the model device
|
||||
# instead of materializing a CPU mel tensor and copying it per request.
|
||||
wav_tensor = torch.from_numpy(wav).to(device=dev, dtype=torch.float32).unsqueeze(0)
|
||||
mel_basis, hann_window = self._get_speaker_mel_buffers(
|
||||
device=dev,
|
||||
sampling_rate=24000,
|
||||
n_fft=1024,
|
||||
num_mels=128,
|
||||
win_size=1024,
|
||||
fmin=0,
|
||||
fmax=12000,
|
||||
)
|
||||
mels = mel_spectrogram(
|
||||
torch.from_numpy(wav).unsqueeze(0),
|
||||
wav_tensor,
|
||||
n_fft=1024,
|
||||
num_mels=128,
|
||||
sampling_rate=24000,
|
||||
@@ -1131,10 +1312,43 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
win_size=1024,
|
||||
fmin=0,
|
||||
fmax=12000,
|
||||
mel_basis=mel_basis,
|
||||
hann_window=hann_window,
|
||||
).transpose(1, 2)
|
||||
spk = self.speaker_encoder(mels.to(dev, dtype=torch.bfloat16))[0]
|
||||
spk = self.speaker_encoder(mels.to(dtype=torch.bfloat16))[0]
|
||||
return spk.to(dtype=torch.bfloat16)
|
||||
|
||||
def _get_speaker_mel_buffers(
|
||||
self,
|
||||
*,
|
||||
device: torch.device,
|
||||
sampling_rate: int,
|
||||
n_fft: int,
|
||||
num_mels: int,
|
||||
win_size: int,
|
||||
fmin: int,
|
||||
fmax: int | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
cache = getattr(self, "_speaker_mel_buffer_cache", None)
|
||||
if cache is None:
|
||||
cache = {}
|
||||
self._speaker_mel_buffer_cache = cache
|
||||
key = (str(device), int(sampling_rate), int(n_fft), int(num_mels), int(win_size), int(fmin), fmax)
|
||||
cached = cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
mel_basis = mel_filter_bank(
|
||||
sr=int(sampling_rate),
|
||||
n_fft=int(n_fft),
|
||||
n_mels=int(num_mels),
|
||||
fmin=int(fmin),
|
||||
fmax=fmax,
|
||||
).to(device)
|
||||
hann_window = torch.hann_window(int(win_size), device=device)
|
||||
cached = (mel_basis, hann_window)
|
||||
cache[key] = cached
|
||||
return cached
|
||||
|
||||
def _ensure_speech_tokenizer_loaded(self) -> Qwen3TTSTokenizer:
|
||||
if self._speech_tokenizer is not None:
|
||||
return self._speech_tokenizer
|
||||
@@ -1182,6 +1396,198 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
return ref_code.to(device=next(self.parameters()).device, dtype=torch.long)
|
||||
raise ValueError("SpeechTokenizer.encode did not return audio_codes tensor")
|
||||
|
||||
@staticmethod
|
||||
def _first_value(value: object, default: object = None) -> object:
|
||||
if isinstance(value, list):
|
||||
return value[0] if value else default
|
||||
return value if value is not None else default
|
||||
|
||||
@staticmethod
|
||||
def _coerce_ref_code_tensor(value: object, *, device: torch.device) -> torch.Tensor | None:
|
||||
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, torch.Tensor):
|
||||
ref_code = value
|
||||
elif isinstance(value, np.ndarray):
|
||||
ref_code = torch.from_numpy(value)
|
||||
elif isinstance(value, list) and value:
|
||||
ref_code = torch.as_tensor(value, dtype=torch.long)
|
||||
else:
|
||||
return None
|
||||
if ref_code.ndim == 3:
|
||||
ref_code = ref_code[0]
|
||||
if ref_code.ndim != 2 or ref_code.numel() == 0:
|
||||
return None
|
||||
return ref_code.to(device=device, dtype=torch.long).contiguous()
|
||||
|
||||
@staticmethod
|
||||
def _coerce_token_ids(value: object, *, device: torch.device) -> torch.Tensor | None:
|
||||
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, torch.Tensor):
|
||||
ids = value
|
||||
elif isinstance(value, np.ndarray):
|
||||
ids = torch.from_numpy(value)
|
||||
elif isinstance(value, list) and value and all(isinstance(v, (int, np.integer)) for v in value):
|
||||
ids = torch.tensor(value, dtype=torch.long)
|
||||
else:
|
||||
return None
|
||||
if ids.ndim == 1:
|
||||
ids = ids.unsqueeze(0)
|
||||
if ids.ndim != 2 or ids.numel() == 0:
|
||||
return None
|
||||
return ids.to(device=device, dtype=torch.long).contiguous()
|
||||
|
||||
@staticmethod
|
||||
def _split_ref_code_batch(ref_codes: object, expected: int) -> list[object] | None:
|
||||
if isinstance(ref_codes, torch.Tensor):
|
||||
if ref_codes.ndim == 3 and int(ref_codes.shape[0]) == expected:
|
||||
return [ref_codes[i] for i in range(expected)]
|
||||
if expected == 1:
|
||||
return [ref_codes]
|
||||
return None
|
||||
if isinstance(ref_codes, (list, tuple)) and len(ref_codes) == expected:
|
||||
return list(ref_codes)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _voice_clone_prompt_dict(raw: object) -> dict[str, object] | None:
|
||||
raw = Qwen3TTSTalkerForConditionalGeneration._first_value(raw)
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
if isinstance(raw, list) and raw and isinstance(raw[0], dict):
|
||||
return raw[0]
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _has_ref_code_like(value: object) -> bool:
|
||||
value = Qwen3TTSTalkerForConditionalGeneration._first_value(value)
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.numel() > 0 and value.ndim in (2, 3)
|
||||
if isinstance(value, np.ndarray):
|
||||
return value.size > 0 and value.ndim in (2, 3)
|
||||
if isinstance(value, list):
|
||||
return bool(value)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _needs_initial_prompt_preprocess(info_dict: dict[str, Any]) -> bool:
|
||||
embed = info_dict.get("embed")
|
||||
return not (isinstance(embed, dict) and "prefill" in embed)
|
||||
|
||||
def _needs_batched_ref_code(self, info_dict: dict[str, Any]) -> bool:
|
||||
if not self._needs_initial_prompt_preprocess(info_dict):
|
||||
return False
|
||||
if self._first_value(info_dict.get("task_type"), "CustomVoice") != "Base":
|
||||
return False
|
||||
codes = info_dict.get("codes")
|
||||
if isinstance(codes, dict) and self._has_ref_code_like(codes.get(_PRECOMPUTED_REF_CODE_KEY)):
|
||||
return False
|
||||
|
||||
voice_clone_prompt = self._voice_clone_prompt_dict(info_dict.get("voice_clone_prompt"))
|
||||
if voice_clone_prompt is not None:
|
||||
if self._has_ref_code_like(voice_clone_prompt.get("ref_code")):
|
||||
return False
|
||||
icl_flag = self._first_value(voice_clone_prompt.get("icl_mode"))
|
||||
if isinstance(icl_flag, bool) and not icl_flag:
|
||||
return False
|
||||
|
||||
xvec_only = self._first_value(info_dict.get("x_vector_only_mode"), False)
|
||||
if isinstance(xvec_only, bool) and xvec_only:
|
||||
return False
|
||||
|
||||
ref_audio_list = info_dict.get("ref_audio")
|
||||
return isinstance(ref_audio_list, list) and bool(ref_audio_list)
|
||||
|
||||
@torch.inference_mode()
|
||||
def preprocess_batch(
|
||||
self,
|
||||
*,
|
||||
req_ids: list[str],
|
||||
model_intermediate_buffer: dict[str, dict[str, Any]],
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""Batch Base voice-clone ref-audio codec extraction for current prefill requests."""
|
||||
pending_text: list[tuple[dict[str, Any], str]] = []
|
||||
pending_ref_text: list[tuple[dict[str, Any], str]] = []
|
||||
groups: dict[int, list[tuple[dict[str, Any], np.ndarray, int]]] = {}
|
||||
for req_id in req_ids:
|
||||
info_dict = model_intermediate_buffer.get(req_id)
|
||||
if not isinstance(info_dict, dict):
|
||||
continue
|
||||
|
||||
if (
|
||||
self._needs_initial_prompt_preprocess(info_dict)
|
||||
and self._first_value(info_dict.get("task_type"), "CustomVoice") == "Base"
|
||||
):
|
||||
if _PRECOMPUTED_TEXT_IDS_KEY not in info_dict:
|
||||
text = self._first_value(info_dict.get("text"), "")
|
||||
if isinstance(text, str):
|
||||
pending_text.append((info_dict, self._build_assistant_text(text)))
|
||||
|
||||
if _PRECOMPUTED_REF_IDS_KEY not in info_dict:
|
||||
ref_text = self._first_value(info_dict.get("ref_text"))
|
||||
if isinstance(ref_text, str) and ref_text.strip():
|
||||
pending_ref_text.append((info_dict, self._build_ref_text(ref_text)))
|
||||
|
||||
if not self._needs_batched_ref_code(info_dict):
|
||||
continue
|
||||
ref_audio_list = info_dict.get("ref_audio")
|
||||
if not isinstance(ref_audio_list, list) or not ref_audio_list:
|
||||
continue
|
||||
try:
|
||||
wav, sr = self._normalize_ref_audio(ref_audio_list[0])
|
||||
except Exception:
|
||||
# Keep the original per-request path responsible for surfacing
|
||||
# invalid ref_audio errors with its existing messages.
|
||||
continue
|
||||
info_dict[_NORMALIZED_REF_AUDIO_KEY] = (wav, sr)
|
||||
groups.setdefault(int(sr), []).append((info_dict, wav, int(sr)))
|
||||
|
||||
if pending_text or pending_ref_text:
|
||||
tok_text = self._get_tokenizer()
|
||||
for items, key in (
|
||||
(pending_text, _PRECOMPUTED_TEXT_IDS_KEY),
|
||||
(pending_ref_text, _PRECOMPUTED_REF_IDS_KEY),
|
||||
):
|
||||
if not items:
|
||||
continue
|
||||
try:
|
||||
tokenized = tok_text([text for _, text in items], padding=False)
|
||||
except Exception as exc:
|
||||
logger.debug("Qwen3-TTS batched text tokenization failed; falling back to serial path: %s", exc)
|
||||
continue
|
||||
input_ids = tokenized.get("input_ids") if isinstance(tokenized, dict) else None
|
||||
if not isinstance(input_ids, list) or len(input_ids) != len(items):
|
||||
continue
|
||||
for (info_dict, _), ids in zip(items, input_ids, strict=True):
|
||||
if isinstance(ids, list) and ids:
|
||||
info_dict[key] = torch.tensor(ids, dtype=torch.long)
|
||||
|
||||
groups = {sr: items for sr, items in groups.items() if len(items) >= 2}
|
||||
if not groups:
|
||||
return
|
||||
|
||||
tok = self._ensure_speech_tokenizer_loaded()
|
||||
for sr, items in groups.items():
|
||||
wavs = [wav for _, wav, _ in items]
|
||||
try:
|
||||
enc = tok.encode(wavs, sr=int(sr), return_dict=True)
|
||||
ref_codes = self._split_ref_code_batch(getattr(enc, "audio_codes", None), len(items))
|
||||
except Exception as exc:
|
||||
logger.debug("Qwen3-TTS batched ref_code encode failed; falling back to serial path: %s", exc)
|
||||
continue
|
||||
if ref_codes is None:
|
||||
continue
|
||||
for (info_dict, wav, item_sr), ref_code in zip(items, ref_codes, strict=True):
|
||||
ref_code_t = self._coerce_ref_code_tensor(ref_code, device=device)
|
||||
if ref_code_t is None:
|
||||
continue
|
||||
info_dict.setdefault("codes", {})[_PRECOMPUTED_REF_CODE_KEY] = ref_code_t
|
||||
info_dict[_NORMALIZED_REF_AUDIO_KEY] = (wav, item_sr)
|
||||
|
||||
def _generate_icl_prompt(
|
||||
self,
|
||||
*,
|
||||
@@ -1256,9 +1662,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
# Text ids for assistant template (always).
|
||||
tok = self._get_tokenizer()
|
||||
input_ids = tok(self._build_assistant_text(text), return_tensors="pt", padding=False)["input_ids"].to(
|
||||
device=next(self.parameters()).device
|
||||
)
|
||||
dev = next(self.parameters()).device
|
||||
input_ids = self._coerce_token_ids(info_dict.pop(_PRECOMPUTED_TEXT_IDS_KEY, None), device=dev)
|
||||
if input_ids is None:
|
||||
input_ids = tok(self._build_assistant_text(text), return_tensors="pt", padding=False)["input_ids"].to(
|
||||
device=dev
|
||||
)
|
||||
|
||||
# Optional instruct prefix.
|
||||
instruct = (info_dict.get("instruct") or [""])[0]
|
||||
@@ -1362,6 +1771,26 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
xvec_only = bool((info_dict.get("x_vector_only_mode") or [False])[0])
|
||||
in_context_mode = not xvec_only
|
||||
voice_clone_prompt = _normalize_voice_clone_prompt(info_dict.get("voice_clone_prompt"))
|
||||
ref_audio_wav: np.ndarray | None = None
|
||||
ref_audio_sr: int | None = None
|
||||
|
||||
def _get_ref_audio() -> tuple[np.ndarray, int]:
|
||||
nonlocal ref_audio_wav, ref_audio_sr
|
||||
if ref_audio_wav is None or ref_audio_sr is None:
|
||||
normalized_ref_audio = info_dict.pop(_NORMALIZED_REF_AUDIO_KEY, None)
|
||||
if (
|
||||
isinstance(normalized_ref_audio, tuple)
|
||||
and len(normalized_ref_audio) == 2
|
||||
and isinstance(normalized_ref_audio[0], np.ndarray)
|
||||
):
|
||||
ref_audio_wav = normalized_ref_audio[0]
|
||||
ref_audio_sr = int(normalized_ref_audio[1])
|
||||
return ref_audio_wav, ref_audio_sr
|
||||
ref_audio_list = info_dict.get("ref_audio")
|
||||
if not isinstance(ref_audio_list, list) or not ref_audio_list:
|
||||
raise ValueError("Base requires `ref_audio`.")
|
||||
ref_audio_wav, ref_audio_sr = self._normalize_ref_audio(ref_audio_list[0])
|
||||
return ref_audio_wav, ref_audio_sr
|
||||
|
||||
# Speaker cache: only for uploaded (named) speakers
|
||||
_speaker_cache_key = None
|
||||
@@ -1413,12 +1842,15 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
ref_code_t = ref_code_t[0]
|
||||
ref_code_t = ref_code_t.to(device=input_ids.device, dtype=torch.long)
|
||||
ref_code_len = int(ref_code_t.shape[0])
|
||||
elif in_context_mode:
|
||||
else:
|
||||
codes = info_dict.get("codes")
|
||||
precomputed_ref_code = codes.get(_PRECOMPUTED_REF_CODE_KEY) if isinstance(codes, dict) else None
|
||||
ref_code_t = self._coerce_ref_code_tensor(precomputed_ref_code, device=input_ids.device)
|
||||
if isinstance(ref_code_t, torch.Tensor):
|
||||
ref_code_len = int(ref_code_t.shape[0])
|
||||
if ref_code_t is None and in_context_mode:
|
||||
# Compute ref_code from ref_audio if not provided.
|
||||
ref_audio_list = info_dict.get("ref_audio")
|
||||
if not isinstance(ref_audio_list, list) or not ref_audio_list:
|
||||
raise ValueError("Base requires `ref_audio`.")
|
||||
wav_np, sr = self._normalize_ref_audio(ref_audio_list[0])
|
||||
wav_np, sr = _get_ref_audio()
|
||||
ref_code_t = self._encode_ref_audio_to_code(wav_np, sr).to(device=input_ids.device)
|
||||
ref_code_len = int(ref_code_t.shape[0])
|
||||
if isinstance(ref_code_t, torch.Tensor):
|
||||
@@ -1437,10 +1869,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
# Plain list/array from API (survived msgspec IPC serialization).
|
||||
speaker_embed = torch.tensor(spk, dtype=torch.bfloat16, device=input_ids.device).view(1, 1, -1)
|
||||
else:
|
||||
ref_audio_list = info_dict.get("ref_audio")
|
||||
if not isinstance(ref_audio_list, list) or not ref_audio_list:
|
||||
raise ValueError("Base requires `ref_audio`.")
|
||||
wav_np, sr = self._normalize_ref_audio(ref_audio_list[0])
|
||||
wav_np, sr = _get_ref_audio()
|
||||
speaker_embed = self._extract_speaker_embedding(wav_np, sr).view(1, 1, -1)
|
||||
|
||||
# Cache miss: store extraction result
|
||||
@@ -1467,7 +1896,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
if in_context_mode:
|
||||
# Prefer explicit tokenized `ref_ids` if provided (matches official signature).
|
||||
ref_ids = _to_long_tensor(info_dict.get("ref_ids"), device=input_ids.device)
|
||||
ref_ids = self._coerce_token_ids(info_dict.pop(_PRECOMPUTED_REF_IDS_KEY, None), device=input_ids.device)
|
||||
if ref_ids is None:
|
||||
ref_ids = _to_long_tensor(info_dict.get("ref_ids"), device=input_ids.device)
|
||||
if ref_ids is None and voice_clone_prompt is not None:
|
||||
ref_ids = _to_long_tensor(
|
||||
voice_clone_prompt.get("ref_ids") or voice_clone_prompt.get("ref_id"), device=input_ids.device
|
||||
|
||||
+39
-1
@@ -852,6 +852,9 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
def enable_cudagraph(
|
||||
self,
|
||||
capture_sizes: list[int] | None = None,
|
||||
capture_batch_sizes: list[int] | None = None,
|
||||
extra_capture_shapes: list[tuple[int, int]] | None = None,
|
||||
compile_shapes: list[tuple[int, int]] | None = None,
|
||||
device: torch.device | None = None,
|
||||
codec_chunk_frames: int = 0,
|
||||
codec_left_context_frames: int = 0,
|
||||
@@ -869,6 +872,9 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
self._cudagraph_wrapper = CUDAGraphDecoderWrapper(
|
||||
decoder=self,
|
||||
capture_sizes=capture_sizes,
|
||||
capture_batch_sizes=capture_batch_sizes,
|
||||
extra_capture_shapes=extra_capture_shapes,
|
||||
compile_shapes=compile_shapes,
|
||||
num_quantizers=self.config.num_quantizers,
|
||||
enabled=True,
|
||||
)
|
||||
@@ -882,8 +888,11 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
)
|
||||
self._cudagraph_enabled = True
|
||||
logger.info(
|
||||
"CUDA Graph enabled for decoder: seq_lens=%s",
|
||||
"CUDA Graph enabled for decoder: batch_sizes=%s seq_lens=%s extra_shapes=%s compile_shapes=%s",
|
||||
self._cudagraph_wrapper.capture_batch_sizes,
|
||||
self._cudagraph_wrapper.capture_sizes,
|
||||
self._cudagraph_wrapper.extra_capture_shapes,
|
||||
self._cudagraph_wrapper.compile_shapes,
|
||||
)
|
||||
|
||||
def disable_cudagraph(self):
|
||||
@@ -925,6 +934,35 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
|
||||
start_index = end_index
|
||||
return torch.cat(wavs, dim=-1)
|
||||
|
||||
def batched_chunked_decode(
|
||||
self,
|
||||
codes,
|
||||
lengths,
|
||||
chunk_size=300,
|
||||
left_context_size=25,
|
||||
max_batch_size=0,
|
||||
):
|
||||
if self._cudagraph_enabled and self._cudagraph_wrapper is not None:
|
||||
return self._cudagraph_wrapper.batched_chunked_decode_with_cudagraph(
|
||||
codes,
|
||||
lengths,
|
||||
chunk_size=chunk_size,
|
||||
left_context_size=left_context_size,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
|
||||
from ..cuda_graph_decoder_wrapper import _batched_chunked_decode
|
||||
|
||||
return _batched_chunked_decode(
|
||||
codes,
|
||||
lengths,
|
||||
decode_fn=self,
|
||||
total_upsample=self.total_upsample,
|
||||
chunk_size=chunk_size,
|
||||
left_context_size=left_context_size,
|
||||
max_batch_size=max_batch_size,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3TTSTokenizerV2Encoder(MimiModel):
|
||||
def __init__(self, config: MimiConfig):
|
||||
|
||||
Reference in new issue
Block a user