[New Model] Support IndexTTS 2.5 (#5957)

Signed-off-by: Sysy <1370724210@qq.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Sy03andYueqian Lin authored and GitHub committed 2026-08-11 23:00:35 +00:00
1 parent 1ff2753469
commit 58007c0084
77 files changed
+7123 -519

No files matched your search

@@ -22,7 +22,7 @@ logger = init_logger(__name__)
# ---------------------------------------------------------------------------
_wav2vec2_cache: dict[tuple[str, str], tuple] = {}
_semantic_codec_cache: dict[tuple[str, str], Any] = {}
_semantic_codec_cache: dict[tuple[str, str, str], Any] = {}
_campplus_cache: dict[tuple[str, str], Any] = {}
_qwen_emotion_cache: dict[tuple[str, str], tuple] = {}
@@ -34,11 +34,40 @@ def _freeze(model):
return model
def local_model_roots(model_path: str) -> tuple[str, ...]:
"""Return native IndexTTS asset roots in lookup order.
HuggingFace-converted checkpoints place assets at the model root, while
the official IndexTTS repository and lab-published bundle keep them under
``checkpoints/``.
"""
roots = [model_path]
checkpoints = os.path.join(model_path, "checkpoints")
if os.path.isdir(checkpoints):
roots.append(checkpoints)
return tuple(roots)
def resolve_model_directory(
model_path: str,
*relative_paths: str,
) -> str | None:
for root in local_model_roots(model_path):
for relative_path in relative_paths:
candidate = os.path.join(root, relative_path)
if os.path.isdir(candidate):
return candidate
return None
def resolve_model_file(model_path: str, filename: str) -> str | None:
"""Resolve an IndexTTS2 asset from a local model dir or HF repo id."""
local_path = os.path.join(model_path, filename)
if os.path.isfile(local_path):
return local_path
for root in local_model_roots(model_path):
local_path = os.path.join(root, filename)
if os.path.isfile(local_path):
return local_path
if os.path.isdir(model_path):
return None
try:
return cached_file(model_path, filename)
except Exception as e:
@@ -53,11 +82,38 @@ def load_wav2vec2(model_path: str, device: torch.device):
from transformers import AutoFeatureExtractor, Wav2Vec2BertModel
model, processor = None, None
# Try local subfolder first (no lock needed), then HF hub with lock.
# Try the layouts shipped by IndexTTS 2 and 2.5 before the Hub fallback.
local_dir = resolve_model_directory(
model_path,
"wav2vec2bert",
"w2v-bert-2.0",
os.path.join("hf_cache", "w2v-bert-2.0"),
)
if local_dir is not None:
try:
processor = AutoFeatureExtractor.from_pretrained(
local_dir,
local_files_only=True,
)
model = Wav2Vec2BertModel.from_pretrained(
local_dir,
local_files_only=True,
)
except Exception as exc:
raise RuntimeError(f"Failed to load local IndexTTS Wav2Vec2-BERT assets from {local_dir!r}") from exc
if local_dir is None and os.path.isdir(model_path):
raise FileNotFoundError(f"IndexTTS Wav2Vec2-BERT assets are missing from local bundle {model_path!r}")
# Try a remote model subfolder, then the public base model.
try:
src_args = dict(pretrained_model_name_or_path=model_path, subfolder="wav2vec2bert")
processor = AutoFeatureExtractor.from_pretrained(**src_args)
model = Wav2Vec2BertModel.from_pretrained(**src_args)
if model is None or processor is None:
src_args = dict(
pretrained_model_name_or_path=model_path,
subfolder="wav2vec2bert",
)
processor = AutoFeatureExtractor.from_pretrained(**src_args)
model = Wav2Vec2BertModel.from_pretrained(**src_args)
except Exception:
try:
with _repo_prefetch_lock("facebook/w2v-bert-2.0"):
@@ -67,16 +123,43 @@ def load_wav2vec2(model_path: str, device: torch.device):
except Exception:
pass
if model is None or processor is None:
raise RuntimeError(f"Could not load Wav2Vec2-BERT from {model_path}/wav2vec2bert or facebook/w2v-bert-2.0")
raise RuntimeError(
f"Could not load Wav2Vec2-BERT from IndexTTS model assets under {model_path!r} or facebook/w2v-bert-2.0"
)
_freeze(model.to(device=device, dtype=torch.float32))
_wav2vec2_cache[cache_key] = (model, processor)
return model, processor
def load_semantic_codec(model_path: str, config: dict, device: torch.device):
cache_key = (model_path, str(device))
def load_semantic_codec(
model_path: str,
config: dict,
device: torch.device,
*,
codec_type: str = "repcodec",
checkpoint_name: str | None = None,
):
cache_key = (model_path, str(device), codec_type)
if cache_key in _semantic_codec_cache:
return _semantic_codec_cache[cache_key]
if codec_type == "enhanced":
from .codec import EnhancedCodec
codec = EnhancedCodec(**config, cfg=config)
checkpoint_name = checkpoint_name or "codec.pth"
ckpt_path = resolve_model_file(model_path, checkpoint_name)
if ckpt_path is None:
raise FileNotFoundError(
f"IndexTTS 2.5 EnhancedCodec checkpoint {checkpoint_name!r} not found in {model_path!r}"
)
codec.load_checkpoint(ckpt_path)
_freeze(codec.to(device=device, dtype=torch.float32))
_semantic_codec_cache[cache_key] = codec
return codec
if codec_type != "repcodec":
raise ValueError(f"Unsupported IndexTTS semantic codec type: {codec_type!r}")
from .utils.maskgct.repcodec_model import RepCodec
codec = RepCodec(
@@ -84,11 +167,14 @@ def load_semantic_codec(model_path: str, config: dict, device: torch.device):
hidden_size=config.get("hidden_size", 1024),
codebook_dim=config.get("codebook_dim", 8),
)
ckpt_path = resolve_model_file(model_path, "semantic_codec.pth")
checkpoint_name = checkpoint_name or "semantic_codec.pth"
ckpt_path = resolve_model_file(model_path, checkpoint_name)
if ckpt_path is not None:
state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
codec.load_state_dict(state, strict=False)
else:
if os.path.isdir(model_path):
raise FileNotFoundError(f"IndexTTS RepCodec checkpoint is missing from local bundle {model_path!r}")
import safetensors.torch
from huggingface_hub import hf_hub_download
@@ -106,8 +192,19 @@ def load_campplus(model_path: str, device: torch.device):
from .utils.campplus.dtdnn import CAMPPlus
campplus = CAMPPlus(feat_dim=80, embedding_size=192)
ckpt_path = resolve_model_file(model_path, "campplus.pth")
ckpt_path = None
for relative_path in (
"campplus.pth",
"campplus_cn_common.bin",
os.path.join("hf_cache", "campplus_cn_common.bin"),
os.path.join("campplus", "campplus_cn_common.bin"),
):
ckpt_path = resolve_model_file(model_path, relative_path)
if ckpt_path is not None:
break
if ckpt_path is None:
if os.path.isdir(model_path):
raise FileNotFoundError(f"IndexTTS CAMPPlus checkpoint is missing from local bundle {model_path!r}")
from huggingface_hub import hf_hub_download
ckpt_path = hf_hub_download("funasr/campplus", filename="campplus_cn_common.bin")
@@ -125,20 +222,29 @@ def load_qwen_emotion(model_path: str, device: torch.device, *, trust_remote_cod
from transformers import AutoModelForCausalLM, AutoTokenizer
subfolder = "qwen0.6bemo4-merge"
local_dir = resolve_model_directory(model_path, subfolder)
source_args = (
{"pretrained_model_name_or_path": local_dir, "local_files_only": True}
if local_dir is not None
else {
"pretrained_model_name_or_path": model_path,
"subfolder": subfolder,
}
)
try:
tokenizer = AutoTokenizer.from_pretrained(
model_path,
subfolder=subfolder,
**source_args,
trust_remote_code=trust_remote_code,
)
model = AutoModelForCausalLM.from_pretrained(
model_path,
subfolder=subfolder,
**source_args,
torch_dtype="float16",
device_map={"": str(device)},
trust_remote_code=trust_remote_code,
)
except Exception as e:
if local_dir is not None:
raise RuntimeError(f"Failed to load local IndexTTS QwenEmotion assets from {local_dir!r}") from e
logger.warning(
"Could not load QwenEmotion from %s/%s: %s. use_emo_text will fall back to default emotion.",
model_path,
@@ -172,9 +278,8 @@ def compute_fbank(wav_16k: torch.Tensor, device: torch.device) -> torch.Tensor:
)
fbank = fbank - fbank.mean(dim=0, keepdim=True)
return fbank.unsqueeze(0).to(device=device) # [1, T, 80]
except (ImportError, RuntimeError, OSError):
logger.warning("torchaudio not available, returning zero fbank")
return torch.zeros(1, 100, 80, device=device)
except (ImportError, RuntimeError, OSError) as exc:
raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc
def wav2vec_extract(