[Model] Suppress silence codec tokens for the first N Qwen3-TTS decod… (#5048)
Signed-off-by: Inesh Reddy <ineshreddy249@gmail.com>
This commit is contained in:
1 parent
718d9a4cd7
commit
cd3867c373
12 files changed
+547
-1
No files matched your search
@@ -85,6 +85,37 @@ def build_instruct_text(instruct: str) -> str:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_x_vector_only(info_dict: dict) -> bool | None:
|
||||
"""Resolve whether a request runs Base voice-clone in x-vector-only mode.
|
||||
|
||||
Mirrors the resolution in :meth:`Qwen3TTSPromptEmbedsBuilder.build_prompt_embeds`
|
||||
for the ``task_type == "Base"`` branch: the ``x_vector_only_mode`` flag, then
|
||||
the ``voice_clone_prompt.icl_mode`` override when present.
|
||||
|
||||
Returns ``None`` when the mode does not apply (any non-Base task), so callers
|
||||
can distinguish "in-context" from "not a voice-clone request at all".
|
||||
"""
|
||||
task_type = first_value(info_dict.get("task_type"), "CustomVoice")
|
||||
if task_type != "Base":
|
||||
return None
|
||||
|
||||
xvec_only = bool((info_dict.get("x_vector_only_mode") or [False])[0])
|
||||
|
||||
raw = info_dict.get("voice_clone_prompt")
|
||||
if isinstance(raw, list):
|
||||
raw = raw[0] if raw else None
|
||||
if isinstance(raw, list) and raw and isinstance(raw[0], dict):
|
||||
raw = raw[0]
|
||||
if isinstance(raw, dict) and "icl_mode" in raw:
|
||||
icl_flag = raw.get("icl_mode")
|
||||
if isinstance(icl_flag, list):
|
||||
icl_flag = icl_flag[0] if icl_flag else None
|
||||
if isinstance(icl_flag, bool):
|
||||
xvec_only = not icl_flag
|
||||
|
||||
return xvec_only
|
||||
|
||||
|
||||
def first_value(value: object, default: object = None) -> object:
|
||||
if isinstance(value, list):
|
||||
return value[0] if value else default
|
||||
|
||||
@@ -30,7 +30,7 @@ from vllm_omni.utils.speaker_cache import (
|
||||
)
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig
|
||||
from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder
|
||||
from .prompt_embeds_builder import PRECOMPUTED_TEXT_IDS_KEY, Qwen3TTSPromptEmbedsBuilder, resolve_x_vector_only
|
||||
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM
|
||||
from .tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Config
|
||||
from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Encoder
|
||||
@@ -421,6 +421,27 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
codec_mask[self._codec_eos_token_id] = True
|
||||
self.register_buffer("_codec_disallowed_mask", ~codec_mask, persistent=False)
|
||||
|
||||
# Silence-region codec tokens (#4966). The talker has real probability mass on
|
||||
# these in the opening decode frames; how many it draws is a sampling outcome,
|
||||
# which is the source of both the leading silence and its seed-to-seed variance.
|
||||
# The vocabulary is checkpoint-specific, so it is derived at load time in
|
||||
# :meth:`_init_silence_mask`; this is only the placeholder.
|
||||
self._silence_ban_frames = max(0, int(getattr(vllm_config.model_config, "silence_ban_frames", 0) or 0))
|
||||
# vLLM only fills sampling_metadata.output_token_ids when penalties, bad
|
||||
# words, or a logits processor need history. compute_logits reads it for
|
||||
# the per-request decode step, so request it when the ban is on.
|
||||
self.logitsprocs_need_output_token_ids = self._silence_ban_frames > 0
|
||||
self.register_buffer("_silence_mask", torch.zeros((vocab,), dtype=torch.bool), persistent=False)
|
||||
|
||||
# Per-request generation mode for the silence ban (#4966). The ban is a
|
||||
# Base voice-clone (x-vector-only) fix; ICL onsets are legitimate audio
|
||||
# and must not be masked. Mode is only knowable at prefill, so it is
|
||||
# recorded in preprocess_batch and read back at decode. The batch order
|
||||
# comes from set_batch_req_ids, which the runner calls each step with
|
||||
# the same req_id ordering that indexes sampling_metadata.
|
||||
self._req_x_vector_only: dict[str, bool | None] = {}
|
||||
self._batch_req_ids: list[str] = []
|
||||
|
||||
# Keys that should stay on GPU in model_intermediate_buffer to avoid
|
||||
# CPU-to-GPU round-trips on every decode step.
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
|
||||
@@ -560,6 +581,48 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
# Mask out invalid codec ids using the pre-built constant buffer.
|
||||
logits = logits.masked_fill(self._codec_disallowed_mask, float("-inf"))
|
||||
|
||||
# Suppress silence-region tokens for the first N decode frames (#4966). The
|
||||
# per-request decode step is the length of its generated-token history.
|
||||
ban_n = self._silence_ban_frames
|
||||
if ban_n > 0 and sampling_metadata is not None:
|
||||
output_token_ids = getattr(sampling_metadata, "output_token_ids", None)
|
||||
if output_token_ids is None or len(output_token_ids) != logits.shape[0]:
|
||||
logger.warning_once(
|
||||
"Silence ban is enabled but output_token_ids (%s) does not match the "
|
||||
"logits batch (%d); the ban will not take effect.",
|
||||
"None" if output_token_ids is None else len(output_token_ids),
|
||||
logits.shape[0],
|
||||
)
|
||||
else:
|
||||
# Both gates are tiny [B] checks, so keep them on CPU: once no
|
||||
# request is both in scope and inside the window, skip the device
|
||||
# work rather than paying a host-to-device copy and a [B, vocab]
|
||||
# mask on every decode frame.
|
||||
#
|
||||
# Scope the ban to x-vector-only requests (#4966). A request whose
|
||||
# mode was never recorded is left alone: masking a request that
|
||||
# cannot be classified is the risk this gate exists to remove.
|
||||
batch_req_ids = self._batch_req_ids
|
||||
if len(batch_req_ids) == logits.shape[0]:
|
||||
in_scope = [self._req_x_vector_only.get(rid) is True for rid in batch_req_ids]
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Silence ban is enabled but the recorded batch req_ids (%d) do not match "
|
||||
"the logits batch (%d); the ban will not take effect.",
|
||||
len(batch_req_ids),
|
||||
logits.shape[0],
|
||||
)
|
||||
in_scope = [False] * logits.shape[0]
|
||||
|
||||
steps = [len(t) for t in output_token_ids]
|
||||
if any(scoped and step < ban_n for scoped, step in zip(in_scope, steps, strict=True)):
|
||||
early = torch.tensor(
|
||||
[scoped and step < ban_n for scoped, step in zip(in_scope, steps, strict=True)],
|
||||
device=logits.device,
|
||||
dtype=torch.bool,
|
||||
).unsqueeze(1)
|
||||
logits = logits.masked_fill(early & self._silence_mask, float("-inf"))
|
||||
|
||||
return logits
|
||||
|
||||
# -------------------- Omni multimodal output plumbing --------------------
|
||||
@@ -944,12 +1007,30 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""Delegate batched preprocess to :class:`Qwen3TTSPromptEmbedsBuilder`."""
|
||||
if self._silence_ban_frames > 0:
|
||||
for req_id in req_ids:
|
||||
info_dict = model_intermediate_buffer.get(req_id)
|
||||
if isinstance(info_dict, dict):
|
||||
self._req_x_vector_only[req_id] = resolve_x_vector_only(info_dict)
|
||||
self._prompt_builder.preprocess_batch(
|
||||
req_ids=req_ids,
|
||||
model_intermediate_buffer=model_intermediate_buffer,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def set_batch_req_ids(self, req_ids: Sequence[str]) -> None:
|
||||
"""Record the current batch's req_ids, in the order that indexes logits.
|
||||
|
||||
Called by the runner after the forward and before ``compute_logits``,
|
||||
with the same ``input_batch`` ordering that ``sampling_metadata``
|
||||
uses. Doubles as the eviction point: any request no longer in the batch
|
||||
has finished or been aborted, so its recorded mode is dropped.
|
||||
"""
|
||||
self._batch_req_ids = list(req_ids)
|
||||
if len(self._req_x_vector_only) > len(self._batch_req_ids):
|
||||
live = set(self._batch_req_ids)
|
||||
self._req_x_vector_only = {k: v for k, v in self._req_x_vector_only.items() if k in live}
|
||||
|
||||
def _encode_ref_audio_batch(self, wavs: list[np.ndarray], sr: int, *, device: torch.device) -> list[torch.Tensor]:
|
||||
fe = self._encoder_feature_extractor
|
||||
target_sr = int(fe.sampling_rate)
|
||||
@@ -1042,6 +1123,7 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
self.encoder.to(device=device, dtype=torch.bfloat16)
|
||||
|
||||
self._init_runtime_buffers()
|
||||
self._init_silence_mask()
|
||||
|
||||
logger.info("Loaded %d weights for Qwen3TTSTalkerForConditionalGeneration", len(loaded))
|
||||
self._build_stacked_codec_embed()
|
||||
@@ -1056,6 +1138,84 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
device=w.device, dtype=w.dtype
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def _init_silence_mask(self) -> None:
|
||||
"""Derive the silence-region codec vocabulary from the checkpoint.
|
||||
|
||||
Encodes silence at several durations and amplitudes and collects the
|
||||
codebook-0 ids the tokenizer emits. These are the tokens the talker draws
|
||||
when it produces leading silence (#4966). Derived rather than hardcoded
|
||||
because the codec vocabulary is checkpoint-specific.
|
||||
"""
|
||||
if self._silence_ban_frames <= 0:
|
||||
return
|
||||
device = next(self.parameters()).device
|
||||
sr = 24000
|
||||
try:
|
||||
rng = np.random.RandomState(0)
|
||||
wavs = [
|
||||
(amp * rng.randn(int(dur * sr))).astype(np.float32)
|
||||
for amp in (0.0, 1e-5, 1e-4, 1e-3)
|
||||
for dur in (0.3, 0.5, 1.0)
|
||||
]
|
||||
codes = self._encode_ref_audio_batch(wavs, sr, device=device)
|
||||
ids: set[int] = set()
|
||||
for code in codes:
|
||||
code = code.detach().cpu()
|
||||
cb0 = code[:, 0] if code.ndim == 2 else code
|
||||
ids.update(int(t) for t in cb0.tolist())
|
||||
except Exception as exc:
|
||||
logger.warning("Could not derive the silence codec vocabulary; disabling the ban: %s", exc)
|
||||
self._silence_ban_frames = 0
|
||||
return
|
||||
vocab = int(self.talker_config.vocab_size)
|
||||
in_range = sorted(t for t in ids if 0 <= t < vocab)
|
||||
num_out_of_range = len(ids) - len(in_range)
|
||||
if num_out_of_range:
|
||||
# Every derived id should be a codebook-0 index for this checkpoint.
|
||||
# Ids outside the talker vocabulary mean the encoder and the talker
|
||||
# disagree on vocabulary size, so the rest of the derivation is not
|
||||
# trustworthy either.
|
||||
logger.warning(
|
||||
"%d of %d derived silence codec tokens fall outside the talker vocabulary "
|
||||
"(size %d); disabling the ban.",
|
||||
num_out_of_range,
|
||||
len(ids),
|
||||
vocab,
|
||||
)
|
||||
self._silence_ban_frames = 0
|
||||
return
|
||||
if not in_range:
|
||||
# An empty derivation would leave an all-False mask: the ban would
|
||||
# suppress nothing while still paying the per-frame check.
|
||||
logger.warning("Derived no silence codec tokens; disabling the ban.")
|
||||
self._silence_ban_frames = 0
|
||||
return
|
||||
# The silence region is a small corner of codebook 0 (12 tokens on
|
||||
# Qwen3-TTS-12Hz-1.7B-Base). A derivation this large means the encode
|
||||
# returned speech-like codes, and masking that much of the vocabulary
|
||||
# would distort generation rather than trim the onset.
|
||||
max_silence_tokens = max(64, vocab // 20)
|
||||
if len(in_range) > max_silence_tokens:
|
||||
logger.warning(
|
||||
"Derived %d silence codec tokens, above the %d sanity limit for a vocabulary of %d; disabling the ban.",
|
||||
len(in_range),
|
||||
max_silence_tokens,
|
||||
vocab,
|
||||
)
|
||||
self._silence_ban_frames = 0
|
||||
return
|
||||
mask = torch.zeros((vocab,), dtype=torch.bool)
|
||||
for token in in_range:
|
||||
mask[token] = True
|
||||
self._silence_mask.copy_(mask.to(self._silence_mask.device))
|
||||
logger.info(
|
||||
"Derived %d silence codec tokens, suppressed for the first %d decode frames: %s",
|
||||
len(in_range),
|
||||
self._silence_ban_frames,
|
||||
in_range,
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def _init_runtime_buffers(self) -> None:
|
||||
"""Populate request-independent runtime buffers from frozen weights.
|
||||
|
||||
Reference in new issue
Block a user