[New Model] Support IndexTTS 2.5 (#5957)
Signed-off-by: Sysy <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
1ff2753469
commit
58007c0084
77 files changed
+7123
-519
No files matched your search
@@ -22,7 +22,7 @@ logger = init_logger(__name__)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_wav2vec2_cache: dict[tuple[str, str], tuple] = {}
|
||||
_semantic_codec_cache: dict[tuple[str, str], Any] = {}
|
||||
_semantic_codec_cache: dict[tuple[str, str, str], Any] = {}
|
||||
_campplus_cache: dict[tuple[str, str], Any] = {}
|
||||
_qwen_emotion_cache: dict[tuple[str, str], tuple] = {}
|
||||
|
||||
@@ -34,11 +34,40 @@ def _freeze(model):
|
||||
return model
|
||||
|
||||
|
||||
def local_model_roots(model_path: str) -> tuple[str, ...]:
|
||||
"""Return native IndexTTS asset roots in lookup order.
|
||||
|
||||
HuggingFace-converted checkpoints place assets at the model root, while
|
||||
the official IndexTTS repository and lab-published bundle keep them under
|
||||
``checkpoints/``.
|
||||
"""
|
||||
roots = [model_path]
|
||||
checkpoints = os.path.join(model_path, "checkpoints")
|
||||
if os.path.isdir(checkpoints):
|
||||
roots.append(checkpoints)
|
||||
return tuple(roots)
|
||||
|
||||
|
||||
def resolve_model_directory(
|
||||
model_path: str,
|
||||
*relative_paths: str,
|
||||
) -> str | None:
|
||||
for root in local_model_roots(model_path):
|
||||
for relative_path in relative_paths:
|
||||
candidate = os.path.join(root, relative_path)
|
||||
if os.path.isdir(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def resolve_model_file(model_path: str, filename: str) -> str | None:
|
||||
"""Resolve an IndexTTS2 asset from a local model dir or HF repo id."""
|
||||
local_path = os.path.join(model_path, filename)
|
||||
if os.path.isfile(local_path):
|
||||
return local_path
|
||||
for root in local_model_roots(model_path):
|
||||
local_path = os.path.join(root, filename)
|
||||
if os.path.isfile(local_path):
|
||||
return local_path
|
||||
if os.path.isdir(model_path):
|
||||
return None
|
||||
try:
|
||||
return cached_file(model_path, filename)
|
||||
except Exception as e:
|
||||
@@ -53,11 +82,38 @@ def load_wav2vec2(model_path: str, device: torch.device):
|
||||
from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
|
||||
|
||||
model, processor = None, None
|
||||
# Try local subfolder first (no lock needed), then HF hub with lock.
|
||||
# Try the layouts shipped by IndexTTS 2 and 2.5 before the Hub fallback.
|
||||
local_dir = resolve_model_directory(
|
||||
model_path,
|
||||
"wav2vec2bert",
|
||||
"w2v-bert-2.0",
|
||||
os.path.join("hf_cache", "w2v-bert-2.0"),
|
||||
)
|
||||
if local_dir is not None:
|
||||
try:
|
||||
processor = AutoFeatureExtractor.from_pretrained(
|
||||
local_dir,
|
||||
local_files_only=True,
|
||||
)
|
||||
model = Wav2Vec2BertModel.from_pretrained(
|
||||
local_dir,
|
||||
local_files_only=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Failed to load local IndexTTS Wav2Vec2-BERT assets from {local_dir!r}") from exc
|
||||
|
||||
if local_dir is None and os.path.isdir(model_path):
|
||||
raise FileNotFoundError(f"IndexTTS Wav2Vec2-BERT assets are missing from local bundle {model_path!r}")
|
||||
|
||||
# Try a remote model subfolder, then the public base model.
|
||||
try:
|
||||
src_args = dict(pretrained_model_name_or_path=model_path, subfolder="wav2vec2bert")
|
||||
processor = AutoFeatureExtractor.from_pretrained(**src_args)
|
||||
model = Wav2Vec2BertModel.from_pretrained(**src_args)
|
||||
if model is None or processor is None:
|
||||
src_args = dict(
|
||||
pretrained_model_name_or_path=model_path,
|
||||
subfolder="wav2vec2bert",
|
||||
)
|
||||
processor = AutoFeatureExtractor.from_pretrained(**src_args)
|
||||
model = Wav2Vec2BertModel.from_pretrained(**src_args)
|
||||
except Exception:
|
||||
try:
|
||||
with _repo_prefetch_lock("facebook/w2v-bert-2.0"):
|
||||
@@ -67,16 +123,43 @@ def load_wav2vec2(model_path: str, device: torch.device):
|
||||
except Exception:
|
||||
pass
|
||||
if model is None or processor is None:
|
||||
raise RuntimeError(f"Could not load Wav2Vec2-BERT from {model_path}/wav2vec2bert or facebook/w2v-bert-2.0")
|
||||
raise RuntimeError(
|
||||
f"Could not load Wav2Vec2-BERT from IndexTTS model assets under {model_path!r} or facebook/w2v-bert-2.0"
|
||||
)
|
||||
_freeze(model.to(device=device, dtype=torch.float32))
|
||||
_wav2vec2_cache[cache_key] = (model, processor)
|
||||
return model, processor
|
||||
|
||||
|
||||
def load_semantic_codec(model_path: str, config: dict, device: torch.device):
|
||||
cache_key = (model_path, str(device))
|
||||
def load_semantic_codec(
|
||||
model_path: str,
|
||||
config: dict,
|
||||
device: torch.device,
|
||||
*,
|
||||
codec_type: str = "repcodec",
|
||||
checkpoint_name: str | None = None,
|
||||
):
|
||||
cache_key = (model_path, str(device), codec_type)
|
||||
if cache_key in _semantic_codec_cache:
|
||||
return _semantic_codec_cache[cache_key]
|
||||
|
||||
if codec_type == "enhanced":
|
||||
from .codec import EnhancedCodec
|
||||
|
||||
codec = EnhancedCodec(**config, cfg=config)
|
||||
checkpoint_name = checkpoint_name or "codec.pth"
|
||||
ckpt_path = resolve_model_file(model_path, checkpoint_name)
|
||||
if ckpt_path is None:
|
||||
raise FileNotFoundError(
|
||||
f"IndexTTS 2.5 EnhancedCodec checkpoint {checkpoint_name!r} not found in {model_path!r}"
|
||||
)
|
||||
codec.load_checkpoint(ckpt_path)
|
||||
_freeze(codec.to(device=device, dtype=torch.float32))
|
||||
_semantic_codec_cache[cache_key] = codec
|
||||
return codec
|
||||
if codec_type != "repcodec":
|
||||
raise ValueError(f"Unsupported IndexTTS semantic codec type: {codec_type!r}")
|
||||
|
||||
from .utils.maskgct.repcodec_model import RepCodec
|
||||
|
||||
codec = RepCodec(
|
||||
@@ -84,11 +167,14 @@ def load_semantic_codec(model_path: str, config: dict, device: torch.device):
|
||||
hidden_size=config.get("hidden_size", 1024),
|
||||
codebook_dim=config.get("codebook_dim", 8),
|
||||
)
|
||||
ckpt_path = resolve_model_file(model_path, "semantic_codec.pth")
|
||||
checkpoint_name = checkpoint_name or "semantic_codec.pth"
|
||||
ckpt_path = resolve_model_file(model_path, checkpoint_name)
|
||||
if ckpt_path is not None:
|
||||
state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
|
||||
codec.load_state_dict(state, strict=False)
|
||||
else:
|
||||
if os.path.isdir(model_path):
|
||||
raise FileNotFoundError(f"IndexTTS RepCodec checkpoint is missing from local bundle {model_path!r}")
|
||||
import safetensors.torch
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
@@ -106,8 +192,19 @@ def load_campplus(model_path: str, device: torch.device):
|
||||
from .utils.campplus.dtdnn import CAMPPlus
|
||||
|
||||
campplus = CAMPPlus(feat_dim=80, embedding_size=192)
|
||||
ckpt_path = resolve_model_file(model_path, "campplus.pth")
|
||||
ckpt_path = None
|
||||
for relative_path in (
|
||||
"campplus.pth",
|
||||
"campplus_cn_common.bin",
|
||||
os.path.join("hf_cache", "campplus_cn_common.bin"),
|
||||
os.path.join("campplus", "campplus_cn_common.bin"),
|
||||
):
|
||||
ckpt_path = resolve_model_file(model_path, relative_path)
|
||||
if ckpt_path is not None:
|
||||
break
|
||||
if ckpt_path is None:
|
||||
if os.path.isdir(model_path):
|
||||
raise FileNotFoundError(f"IndexTTS CAMPPlus checkpoint is missing from local bundle {model_path!r}")
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
ckpt_path = hf_hub_download("funasr/campplus", filename="campplus_cn_common.bin")
|
||||
@@ -125,20 +222,29 @@ def load_qwen_emotion(model_path: str, device: torch.device, *, trust_remote_cod
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
subfolder = "qwen0.6bemo4-merge"
|
||||
local_dir = resolve_model_directory(model_path, subfolder)
|
||||
source_args = (
|
||||
{"pretrained_model_name_or_path": local_dir, "local_files_only": True}
|
||||
if local_dir is not None
|
||||
else {
|
||||
"pretrained_model_name_or_path": model_path,
|
||||
"subfolder": subfolder,
|
||||
}
|
||||
)
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_path,
|
||||
subfolder=subfolder,
|
||||
**source_args,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_path,
|
||||
subfolder=subfolder,
|
||||
**source_args,
|
||||
torch_dtype="float16",
|
||||
device_map={"": str(device)},
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
except Exception as e:
|
||||
if local_dir is not None:
|
||||
raise RuntimeError(f"Failed to load local IndexTTS QwenEmotion assets from {local_dir!r}") from e
|
||||
logger.warning(
|
||||
"Could not load QwenEmotion from %s/%s: %s. use_emo_text will fall back to default emotion.",
|
||||
model_path,
|
||||
@@ -172,9 +278,8 @@ def compute_fbank(wav_16k: torch.Tensor, device: torch.device) -> torch.Tensor:
|
||||
)
|
||||
fbank = fbank - fbank.mean(dim=0, keepdim=True)
|
||||
return fbank.unsqueeze(0).to(device=device) # [1, T, 80]
|
||||
except (ImportError, RuntimeError, OSError):
|
||||
logger.warning("torchaudio not available, returning zero fbank")
|
||||
return torch.zeros(1, 100, 80, device=device)
|
||||
except (ImportError, RuntimeError, OSError) as exc:
|
||||
raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc
|
||||
|
||||
|
||||
def wav2vec_extract(
|
||||
|
||||
Reference in new issue
Block a user