[Qwen3TTS] [TTS] [Feat] Refactor voice cache manager (#2108)
Signed-off-by: JuanPZuluaga <juanz9312@gmal.com> Signed-off-by: yiliu30 <yi4.liu@intel.com> Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Binh Tang <tangbinhna@gmail.com> Signed-off-by: Binh Tang <binht@netflix.com> Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Signed-off-by: Rein Yang <ruiruyang2@gmail.com> Signed-off-by: CHEN <116010019@link.cuhk.edu.cn> Signed-off-by: vraiti <vraiti@redhat.com> Signed-off-by: Songrui625 <songrui625@gmail.com> Signed-off-by: Lidang Jiang <lidangjiang@gmail.com> Signed-off-by: Lidang-Jiang <lidangjiang@gmail.com> Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: JuanPZuluaga <juanz9312@gmal.com> Co-authored-by: Yi Liu <yi4.liu@intel.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com> Co-authored-by: Binh Tang <tangbinhna@gmail.com> Co-authored-by: Binh Tang <binht@netflix.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com> Co-authored-by: zhumingjue138 <zhumingjue@huawei.com> Co-authored-by: ChenWenjing <54166744+Shirley125@users.noreply.github.com> Co-authored-by: vraiti <vraiti@redhat.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Sy03 <1370724210@qq.com> Co-authored-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Jerry Song <46962917+Songrui625@users.noreply.github.com> Co-authored-by: Lidang Jiang <119769478+Lidang-Jiang@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: linyueqian <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
11 files changed
+442
-614
No files matched your search
@@ -27,6 +27,7 @@ from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer,
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.utils.voice_cache import VoiceEmbeddingCache
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig
|
||||
from .qwen3_tts_code_predictor_vllm import Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM
|
||||
@@ -406,6 +407,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
self._tokenizer = None
|
||||
self._speech_tokenizer: Qwen3TTSTokenizer | None = None
|
||||
|
||||
# In-memory LRU cache for voice extraction artifacts (Base voice clone).
|
||||
self._voice_cache = VoiceEmbeddingCache()
|
||||
|
||||
# -------------------- vLLM required hooks --------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
@@ -1326,6 +1330,25 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
xvec_only = bool((info_dict.get("x_vector_only_mode") or [False])[0])
|
||||
in_context_mode = not xvec_only
|
||||
voice_clone_prompt = _normalize_voice_clone_prompt(info_dict.get("voice_clone_prompt"))
|
||||
|
||||
# Voice cache: only for uploaded voices (created_at > 0)
|
||||
_voice_cache_key = None
|
||||
if voice_clone_prompt is None:
|
||||
_speaker_list = info_dict.get("speaker")
|
||||
if isinstance(_speaker_list, list) and _speaker_list:
|
||||
_voice_name = str(_speaker_list[0]).lower()
|
||||
_voice_created_at = float((info_dict.get("voice_created_at") or [0])[0])
|
||||
if _voice_created_at > 0:
|
||||
_voice_cache_key = self._voice_cache.make_cache_key(_voice_name, xvec_only, _voice_created_at)
|
||||
_cached = self._voice_cache.get(_voice_cache_key) if _voice_cache_key is not None else None
|
||||
if _cached is not None:
|
||||
voice_clone_prompt = {
|
||||
"ref_code": _cached.get("ref_code"),
|
||||
"ref_spk_embedding": _cached.get("ref_spk_embedding"),
|
||||
"icl_mode": _cached.get("icl_mode"),
|
||||
}
|
||||
_voice_cache_key = None # hit -> don't store again
|
||||
|
||||
# Official implementation may pass `voice_clone_prompt.icl_mode`.
|
||||
if voice_clone_prompt is not None and "icl_mode" in voice_clone_prompt:
|
||||
icl_flag = _as_singleton(voice_clone_prompt.get("icl_mode"))
|
||||
@@ -1375,6 +1398,19 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
wav_np, sr = self._normalize_ref_audio(ref_audio_list[0])
|
||||
speaker_embed = self._extract_speaker_embedding(wav_np, sr).view(1, 1, -1)
|
||||
|
||||
# Cache miss: store extraction result
|
||||
if _voice_cache_key is not None and speaker_embed is not None:
|
||||
self._voice_cache.put(
|
||||
_voice_cache_key,
|
||||
{
|
||||
"ref_code": ref_code_prompt.detach().cpu()
|
||||
if isinstance(ref_code_prompt, torch.Tensor)
|
||||
else None,
|
||||
"ref_spk_embedding": speaker_embed.detach().cpu().reshape(-1),
|
||||
"icl_mode": in_context_mode,
|
||||
},
|
||||
)
|
||||
|
||||
codec_input = torch.cat([codec_input_0, speaker_embed, codec_input_1], dim=1)
|
||||
|
||||
# Role header (<|im_start|>assistant\n) -> projected text embeds.
|
||||
|
||||
@@ -1,271 +0,0 @@
|
||||
# Copyright 2026 The Alibaba Qwen team.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.entrypoints.openai.metadata_manager import MetadataManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceClonePromptItem:
|
||||
"""
|
||||
Container for one sample's voice-clone prompt information that can be fed to the model.
|
||||
|
||||
Fields are aligned with `Qwen3TTSForConditionalGeneration.generate(..., voice_clone_prompt=...)`.
|
||||
"""
|
||||
|
||||
ref_code: torch.Tensor | None # (T, Q) or (T,) depending on tokenizer 25Hz/12Hz
|
||||
ref_spk_embedding: torch.Tensor # (D,)
|
||||
x_vector_only_mode: bool
|
||||
icl_mode: bool
|
||||
ref_text: str | None = None
|
||||
|
||||
|
||||
class VoiceCacheManager:
|
||||
"""
|
||||
Voice cache manager, responsible for managing custom voice cache functionality.
|
||||
|
||||
Main features:
|
||||
1. Load uploaded speaker information from metadata.json
|
||||
2. Manage voice clone prompt cache
|
||||
3. Update cache status to metadata.json
|
||||
|
||||
Security properties:
|
||||
- No pickle / torch.load
|
||||
- Safetensors-only
|
||||
- Cache path confined to voice samples directory
|
||||
"""
|
||||
|
||||
def __init__(self, speech_voice_samples_dir: str | None = None, metadata_manager: MetadataManager | None = None):
|
||||
"""
|
||||
Initialize the voice cache manager.
|
||||
|
||||
Args:
|
||||
speech_voice_samples_dir: Speech voice samples directory path,
|
||||
if None, get from environment variable
|
||||
metadata_manager: Optional MetadataManager instance for shared metadata access.
|
||||
If not provided, will create its own (less efficient).
|
||||
"""
|
||||
self.speech_voice_samples_dir = speech_voice_samples_dir or os.environ.get(
|
||||
"SPEECH_VOICE_SAMPLES", "/tmp/voice_samples"
|
||||
)
|
||||
|
||||
# Initialize metadata manager
|
||||
if metadata_manager is not None:
|
||||
self.metadata_manager = metadata_manager
|
||||
else:
|
||||
metadata_file = Path(self.speech_voice_samples_dir) / "metadata.json"
|
||||
self.metadata_manager = MetadataManager(metadata_file)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Metadata helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_uploaded_speakers_from_metadata(self) -> dict[str, Any] | None:
|
||||
"""Load uploaded speakers from metadata manager."""
|
||||
try:
|
||||
return self.metadata_manager.get_uploaded_speakers()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load uploaded speakers from metadata: {e}")
|
||||
return None
|
||||
|
||||
def update_metadata_cache_info(self, speaker: str, cache_file_path: Path, status: str = "ready") -> bool:
|
||||
"""
|
||||
Update cache information using metadata manager.
|
||||
|
||||
Args:
|
||||
speaker: Speaker name
|
||||
cache_file_path: Cache file path
|
||||
status: Cache status, default is "ready"
|
||||
|
||||
Returns:
|
||||
bool: Whether the update was successful
|
||||
"""
|
||||
try:
|
||||
speaker_key = speaker.lower()
|
||||
return self.metadata_manager.update_cache_info(
|
||||
speaker_key=speaker_key, cache_file_path=cache_file_path, status=status
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update metadata cache info: {e}")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cache save (SAFE)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def save_voice_cache(
|
||||
self,
|
||||
speaker: str,
|
||||
audio_file_path: Path,
|
||||
prompt_items: list[VoiceClonePromptItem],
|
||||
) -> bool:
|
||||
"""
|
||||
Save voice cache using safetensors (no pickle, no RCE).
|
||||
"""
|
||||
try:
|
||||
cache_file_path = audio_file_path.with_suffix(".safetensors")
|
||||
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
metadata: dict[str, str] = {}
|
||||
|
||||
tensors["__len__"] = torch.tensor(len(prompt_items), dtype=torch.int64)
|
||||
|
||||
for i, item in enumerate(prompt_items):
|
||||
prefix = f"item_{i}_"
|
||||
|
||||
tensors[prefix + "ref_spk_embedding"] = item.ref_spk_embedding.detach().cpu()
|
||||
|
||||
has_ref_code = item.ref_code is not None
|
||||
tensors[prefix + "has_ref_code"] = torch.tensor(int(has_ref_code), dtype=torch.int8)
|
||||
|
||||
if has_ref_code:
|
||||
tensors[prefix + "ref_code"] = item.ref_code.detach().cpu()
|
||||
|
||||
tensors[prefix + "x_vector_only_mode"] = torch.tensor(int(item.x_vector_only_mode), dtype=torch.int8)
|
||||
tensors[prefix + "icl_mode"] = torch.tensor(int(item.icl_mode), dtype=torch.int8)
|
||||
|
||||
if item.ref_text is not None:
|
||||
metadata[prefix + "ref_text"] = item.ref_text
|
||||
|
||||
save_file(tensors, str(cache_file_path), metadata=metadata)
|
||||
|
||||
return self.update_metadata_cache_info(
|
||||
speaker=speaker,
|
||||
cache_file_path=cache_file_path,
|
||||
status="ready",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save safetensors cache for speaker {speaker}: {e}")
|
||||
self.update_metadata_cache_info(speaker, Path(""), "failed")
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cache load (SAFE)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_cached_voice_prompt(
|
||||
self,
|
||||
speaker: str,
|
||||
device: str | None = None,
|
||||
) -> list[VoiceClonePromptItem] | None:
|
||||
"""
|
||||
Load cached VoiceClonePromptItem list from safetensors.
|
||||
"""
|
||||
try:
|
||||
uploaded_speakers = self.load_uploaded_speakers_from_metadata()
|
||||
if not uploaded_speakers:
|
||||
return None
|
||||
|
||||
speaker_key = speaker.lower()
|
||||
if speaker_key not in uploaded_speakers:
|
||||
return None
|
||||
|
||||
speaker_info = uploaded_speakers[speaker_key]
|
||||
if speaker_info.get("cache_status") != "ready":
|
||||
return None
|
||||
|
||||
cache_file_path = Path(speaker_info.get("cache_file", "")).resolve()
|
||||
|
||||
base_dir = Path(self.speech_voice_samples_dir).resolve()
|
||||
|
||||
# ---- Path confinement (critical security check)
|
||||
if not str(cache_file_path).startswith(str(base_dir)):
|
||||
logger.error(f"Illegal cache path outside base dir: {cache_file_path}")
|
||||
return None
|
||||
|
||||
if not cache_file_path.exists():
|
||||
return None
|
||||
|
||||
if cache_file_path.suffix != ".safetensors":
|
||||
logger.error(f"Legacy or unsafe cache format rejected: {cache_file_path}")
|
||||
return None
|
||||
|
||||
with safe_open(cache_file_path, framework="pt", device="cpu") as f:
|
||||
meta = f.metadata()
|
||||
|
||||
num_items = int(f.get_tensor("__len__").item())
|
||||
result: list[VoiceClonePromptItem] = []
|
||||
|
||||
for i in range(num_items):
|
||||
prefix = f"item_{i}_"
|
||||
|
||||
has_ref_code = bool(f.get_tensor(prefix + "has_ref_code").item())
|
||||
|
||||
ref_code = f.get_tensor(prefix + "ref_code").to(device) if has_ref_code else None
|
||||
|
||||
ref_spk_embedding = f.get_tensor(prefix + "ref_spk_embedding").to(device)
|
||||
|
||||
x_vector_only_mode = bool(f.get_tensor(prefix + "x_vector_only_mode").item())
|
||||
icl_mode = bool(f.get_tensor(prefix + "icl_mode").item())
|
||||
|
||||
ref_text = meta.get(prefix + "ref_text")
|
||||
|
||||
result.append(
|
||||
VoiceClonePromptItem(
|
||||
ref_code=ref_code,
|
||||
ref_spk_embedding=ref_spk_embedding,
|
||||
x_vector_only_mode=x_vector_only_mode,
|
||||
icl_mode=icl_mode,
|
||||
ref_text=ref_text,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Safetensors cache loaded for speaker: {speaker}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load safetensors cache for speaker {speaker}: {e}")
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Audio path helper
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_speaker_audio_path(self, speaker: str) -> Path | None:
|
||||
"""
|
||||
Get speaker's audio file path.
|
||||
|
||||
Args:
|
||||
speaker: Speaker name
|
||||
|
||||
Returns:
|
||||
Optional[Path]: Audio file path, returns None if speaker doesn't exist
|
||||
"""
|
||||
uploaded_speakers = self.load_uploaded_speakers_from_metadata()
|
||||
if not uploaded_speakers:
|
||||
return None
|
||||
|
||||
speaker_key = speaker.lower()
|
||||
if speaker_key not in uploaded_speakers:
|
||||
return None
|
||||
|
||||
audio_file_path = Path(uploaded_speakers[speaker_key]["file_path"])
|
||||
if audio_file_path.exists():
|
||||
return audio_file_path
|
||||
|
||||
logger.warning(f"Audio file not found for speaker {speaker}: {audio_file_path}")
|
||||
return None
|
||||
Reference in new issue
Block a user