[New Model] Support IndexTTS 2.5 (#5957)

Signed-off-by: Sysy <1370724210@qq.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Sy03andYueqian Lin authored and GitHub committed 2026-08-11 23:00:35 +00:00
1 parent 1ff2753469
commit 58007c0084
77 files changed
+7123 -519

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
import yaml
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
from vllm_omni.model_executor.models.indextts2.configuration_indextts2 import (
IndexTTS2Config,
IndexTTS25Config,
)
from vllm_omni.model_executor.models.indextts2.pipeline import INDEXTTS25_PIPELINE
def test_indextts25_defaults_are_distinct_from_v2():
v2 = IndexTTS2Config()
v25 = IndexTTS25Config()
assert v2.model_type == "indextts2"
assert v2.gpt["number_text_tokens"] == 12000
assert v2.gpt["condition_num_latent"] == 32
assert v2.use_gpt_latent is True
assert v2.semantic_codec_type == "repcodec"
assert v2.vocab_size == v2.gpt["number_mel_codes"]
assert v25.model_type == "indextts2_5"
assert v25.gpt["number_text_tokens"] == 60509
assert v25.gpt["condition_num_latent"] == 32
assert v25.use_gpt_latent is False
assert v25.semantic_codec_type == "enhanced"
assert v25.semantic_codec_checkpoint == "codec.pth"
assert v25.tokenizer_file == "multilingual_zh_ja_yue_char_del.tiktoken"
assert v25.vocab_size == v25.gpt["number_mel_codes"]
def test_indextts25_explicit_overrides_win_over_defaults():
config = IndexTTS25Config(
use_gpt_latent=True,
gpt={
"number_text_tokens": 70000,
"condition_num_latent": 2,
},
vocab_size=9000,
)
assert config.use_gpt_latent is True
assert config.gpt["number_text_tokens"] == 70000
assert config.gpt["condition_num_latent"] == 2
assert config.gpt["model_dim"] == config.hidden_size
assert config.vocab_size == 9000
def test_indextts25_pipeline_is_registered_with_two_distinct_stages():
assert OMNI_PIPELINES["indextts2_5"] is INDEXTTS25_PIPELINE
assert INDEXTTS25_PIPELINE.default_deploy_config_name == "indextts2_5.yaml"
assert [stage.model_stage for stage in INDEXTTS25_PIPELINE.stages] == [
"indextts2_5_talker",
"indextts2_5_s2mel_decoder",
]
assert [stage.model_arch for stage in INDEXTTS25_PIPELINE.stages] == [
None,
"IndexTTS25S2MelDecoder",
]
assert all("tokenizer" not in stage.extras for stage in INDEXTTS25_PIPELINE.stages)
def test_indextts25_default_deploy_selects_validated_triton_backend():
deploy_path = Path(__file__).parents[4] / "vllm_omni" / "deploy" / "indextts2_5.yaml"
config = yaml.safe_load(deploy_path.read_text(encoding="utf-8"))
stage0 = next(stage for stage in config["stages"] if stage["stage_id"] == 0)
assert stage0["attention_backend"] == "TRITON_ATTN"
assert stage0["enable_chunked_prefill"] is False
def test_indextts25_deploy_configs_keep_latent_policy_in_sync():
deploy_dir = Path(__file__).parents[4] / "vllm_omni" / "deploy"
expected = {"indextts2_5.yaml": False}
for filename, use_gpt_latent in expected.items():
with open(deploy_dir / filename, encoding="utf-8") as deploy_file:
config = yaml.safe_load(deploy_file)
assert config["pipeline"] == "indextts2_5"
assert config["async_chunk"] is False
assert all("tokenizer" not in stage for stage in config["stages"])
assert [stage["hf_overrides"]["use_gpt_latent"] for stage in config["stages"]] == [
use_gpt_latent,
use_gpt_latent,
]
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm_omni.model_executor.models.indextts2.dit_cuda_graph import (
CUDAGraphDiTRunner,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _CopySpy:
def __init__(self):
self.sources = []
def copy_(self, source):
self.sources.append(source)
return self
class _GraphSpy:
def __init__(self):
self.replays = 0
def replay(self):
self.replays += 1
def test_replay_only_updates_dynamic_inputs_and_reuses_static_output():
runner = CUDAGraphDiTRunner(transformer=object())
graph = _GraphSpy()
static_x = _CopySpy()
static_c = _CopySpy()
static_pos = _CopySpy()
static_mask = _CopySpy()
static_out = object()
entry = SimpleNamespace(
graph=graph,
static_x=static_x,
static_c=static_c,
static_pos=static_pos,
static_mask=static_mask,
static_out=static_out,
)
x_in = object()
c = object()
output = runner._replay(entry, x_in=x_in, c=c)
assert static_x.sources == [x_in]
assert static_c.sources == [c]
assert static_pos.sources == []
assert static_mask.sources == []
assert graph.replays == 1
assert output is static_out
def test_graph_stats_start_empty_and_remain_bounded():
runner = CUDAGraphDiTRunner(transformer=object(), max_graphs=4)
stats = runner.stats_snapshot()
stats["hits"] = 99
assert runner.stats_snapshot() == {
"calls": 0,
"hits": 0,
"captures": 0,
"capture_failures": 0,
"evictions": 0,
"eager": 0,
"cache_size": 0,
}
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm_omni.model_executor.models.indextts2 import codec as codec_module
from vllm_omni.model_executor.models.indextts2 import preprocess_utils
from vllm_omni.model_executor.models.indextts2.codec import EnhancedCodec
def test_enhanced_codec_decode_restores_hidden_size_and_doubles_time():
codec = EnhancedCodec(
codebook_size=8,
hidden_size=4,
codebook_dim=2,
vocos_dim=4,
vocos_intermediate_dim=8,
vocos_num_layers=1,
downsample_scale=2,
).eval()
decoded = codec.decode(torch.tensor([[0, 1, 2]]))
assert decoded.shape == (1, 6, 4)
def test_enhanced_codec_inference_module_does_not_allocate_unused_encoder():
codec = EnhancedCodec(
codebook_size=8,
hidden_size=4,
codebook_dim=2,
vocos_dim=4,
vocos_intermediate_dim=8,
vocos_num_layers=1,
)
parameter_names = set(dict(codec.named_parameters()))
assert not any(name.startswith("encoder.") for name in parameter_names)
assert not any(name.startswith("down.") for name in parameter_names)
assert any(name.startswith("decoder.") for name in parameter_names)
assert any(name.startswith("quantizer.") for name in parameter_names)
def test_semantic_codec_loader_selects_enhanced_checkpoint(tmp_path, monkeypatch):
checkpoint = tmp_path / "codec.pth"
checkpoint.touch()
seen = {}
class FakeCodec:
def load_checkpoint(self, path):
seen["path"] = path
def to(self, **kwargs):
seen["to"] = kwargs
return self
def eval(self):
return self
def parameters(self):
return ()
monkeypatch.setattr(codec_module, "EnhancedCodec", lambda **kwargs: FakeCodec())
preprocess_utils._semantic_codec_cache.clear()
codec = preprocess_utils.load_semantic_codec(
str(tmp_path),
{"codebook_size": 8192},
torch.device("cpu"),
codec_type="enhanced",
checkpoint_name="codec.pth",
)
assert isinstance(codec, FakeCodec)
assert seen["path"] == str(checkpoint)
assert seen["to"] == {"device": torch.device("cpu"), "dtype": torch.float32}
def test_model_asset_resolver_supports_native_checkpoints_layout(tmp_path):
checkpoint = tmp_path / "checkpoints" / "codec.pth"
checkpoint.parent.mkdir()
checkpoint.touch()
assert preprocess_utils.resolve_model_file(str(tmp_path), "codec.pth") == str(checkpoint)
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
from pathlib import Path
from types import SimpleNamespace
import pytest
from vllm_omni.model_executor.models.indextts2 import prompt_utils
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
INDEXTTS25_TOKENIZER_FILE,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _load_indextts2_offline_example():
example_path = (
Path(__file__).resolve().parents[4]
/ "examples"
/ "offline_inference"
/ "text_to_speech"
/ "indextts2"
/ "end2end.py"
)
spec = importlib.util.spec_from_file_location(
"indextts2_offline_example_test",
example_path,
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_offline_main_uses_resolved_stage0_tokenizer_file(monkeypatch, tmp_path):
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
example = _load_indextts2_offline_example()
stage0 = SimpleNamespace(engine_args=SimpleNamespace(hf_overrides={"tokenizer_file": "custom-tokenizer.tiktoken"}))
fake_omni = SimpleNamespace(
stage_configs=[stage0],
generate=lambda *args, **kwargs: [],
)
captured = {}
monkeypatch.setattr(example, "Omni", lambda **kwargs: fake_omni)
monkeypatch.setattr(example, "SamplingParams", lambda **kwargs: kwargs)
monkeypatch.setattr(
example,
"build_request",
lambda **kwargs: captured.update(kwargs) or {"additional_information": {}},
)
example.main(
SimpleNamespace(
model="/model",
model_version="2.5",
deploy_config="/deploy/custom.yaml",
use_gpt_latent=False,
stage_init_timeout=300,
seed=42,
output_dir=str(tmp_path),
text="hello",
ref_audio=None,
emo_audio=None,
emo_text=None,
emo_vector=None,
emo_alpha=None,
use_emo_text=False,
use_random=False,
lang="en",
text_normalization=True,
)
)
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
def test_indextts25_prompt_len_uses_three_conditioning_tokens(monkeypatch):
captured = {}
def fake_prepare(*args, **kwargs):
captured.update(kwargs)
return [11, 12, 13, 14], 0
monkeypatch.setattr(
prompt_utils,
"prepare_indextts25_text",
fake_prepare,
)
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
"/model",
"hello",
model_type="indextts2_5",
lang="en",
text_normalization=False,
)
# Official prepare_gpt_inputs wraps the language-prefixed tokenizer IDs
# with start_text=0 and stop_text=1 before appending start_mel.
assert prompt_len == 3 + (4 + 2) + 1
assert captured["text_normalization"] is False
assert captured["tokenizer_file"] == INDEXTTS25_TOKENIZER_FILE
def test_indextts25_prompt_len_uses_custom_tokenizer_file(monkeypatch):
captured = {}
def fake_prepare(*args, **kwargs):
captured.update(kwargs)
return [11, 12], 0
monkeypatch.setattr(
prompt_utils,
"prepare_indextts25_text",
fake_prepare,
)
prompt_utils.estimate_indextts2_prefill_prompt_len(
"/model",
"hello",
model_type="indextts2_5",
tokenizer_file="custom-tokenizer.tiktoken",
)
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
def test_indextts25_prompt_len_filters_existing_text_wrapper_ids(monkeypatch):
monkeypatch.setattr(
prompt_utils,
"prepare_indextts25_text",
lambda *args, **kwargs: ([0, 58838, 1, 42, 0], 0),
)
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
"/model",
"hello",
model_type="indextts2_5",
lang="en",
)
assert prompt_len == 3 + (2 + 2) + 1
def test_indextts2_prompt_len_ignores_v25_tokenizer_file(monkeypatch):
class FakeTokenizer:
def encode(self, text, *, add_special_tokens):
assert text == "hello"
assert add_special_tokens is False
return [11, 12]
def fail_prepare(*args, **kwargs):
raise AssertionError("IndexTTS 2 must not use the IndexTTS 2.5 tokenizer")
monkeypatch.setattr(
prompt_utils,
"_get_text_tokenizer",
lambda model_id_or_path: FakeTokenizer(),
)
monkeypatch.setattr(
prompt_utils,
"prepare_indextts25_text",
fail_prepare,
)
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
"/model",
"hello",
model_type="indextts2",
tokenizer_file="custom.tiktoken",
)
assert prompt_len == 34 + (2 + 2) + 1
def test_indextts25_prompt_ids_preserve_placeholder_value(monkeypatch):
captured = {}
def fake_estimate(*args, **kwargs):
captured.update(kwargs)
return 10
monkeypatch.setattr(
prompt_utils,
"estimate_indextts2_prefill_prompt_len",
fake_estimate,
)
prompt_ids = prompt_utils.build_indextts2_prefill_prompt_ids(
"/model",
"hello",
model_type="indextts2_5",
lang="en",
tokenizer_file="custom-tokenizer.tiktoken",
placeholder_token_id=17,
)
assert prompt_ids == [17] * 10
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
@@ -0,0 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
import yaml
def test_indextts25_default_recipe_enables_validated_cfm_batching():
recipe_path = Path(__file__).parents[4] / "vllm_omni" / "deploy" / "indextts2_5.yaml"
recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8"))
stage1 = next(stage for stage in recipe["stages"] if stage["stage_id"] == 1)
assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.stage_input_processors.indextts2 import (
STOP_MEL_TOKEN,
talker2s2mel_full_payload,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_no_latent_full_payload_uses_sampled_output_ids_and_strips_stop():
request = SimpleNamespace(
request_id="req-1",
output_token_ids=[17, 23, STOP_MEL_TOKEN],
sampling_params=SimpleNamespace(seed=42),
)
payload = {
"meta": {
"use_gpt_latent": False,
"S_ref": torch.ones(1, 4),
"ref_mel": torch.ones(1, 80, 3),
"style": torch.ones(1, 192),
}
}
result = talker2s2mel_full_payload(None, payload, request)
assert result is not None
assert result["mel_codes"].tolist() == [[17, 23]]
assert result["code_lens"].tolist() == [2]
assert result["use_gpt_latent"] is False
assert result["seed"] == 42
def test_no_latent_full_payload_rejects_legacy_payload_codes():
request = SimpleNamespace(
request_id="legacy",
output_token_ids=[],
sampling_params=SimpleNamespace(seed=None),
)
payload = {
"codes": {"mel": torch.tensor([31, 37, STOP_MEL_TOKEN])},
"meta": {"use_gpt_latent": False},
}
with pytest.raises(ValueError, match="no completed output_token_ids"):
talker2s2mel_full_payload(None, payload, request)
@@ -0,0 +1,476 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config import CompilationMode
from vllm_omni.model_executor.models.indextts2.indextts2_talker import (
IndexTTS2TalkerForConditionalGeneration,
_normalize_indextts_checkpoint_key,
_require_exact_indextts_prefill_span,
resolve_indextts_conditioning_policy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_v25_conditioning_policy():
policy = resolve_indextts_conditioning_policy("indextts2_5")
assert policy.prefix_tokens == 3
assert policy.uses_conformer_perceiver is False
assert policy.uses_campplus_projection is True
assert policy.uses_language_embedding is True
def test_v2_conditioning_policy_is_unchanged():
policy = resolve_indextts_conditioning_policy("indextts2")
assert policy.prefix_tokens == 34
assert policy.uses_conformer_perceiver is True
assert policy.uses_campplus_projection is False
assert policy.uses_language_embedding is False
def test_v25_text_tokens_wrap_language_prefixed_ids_with_start_and_stop(
monkeypatch,
):
import vllm_omni.model_executor.models.indextts2.text_processing_v2_5 as text_processing
monkeypatch.setattr(
text_processing,
"prepare_indextts25_text",
lambda *args, **kwargs: ([58838, 42], 7),
)
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
talker.conditioning_policy = resolve_indextts_conditioning_policy("indextts2_5")
talker.model_path = "/model"
talker.config = SimpleNamespace(tokenizer_file="tokenizer.tiktoken")
token_ids, language_id = talker._tokenize_text(
"hello",
torch.device("cpu"),
lang="ja",
)
assert token_ids.tolist() == [[0, 58838, 42, 1]]
assert language_id.tolist() == [7]
def test_v25_text_tokens_filter_existing_start_and_stop_ids(monkeypatch):
import vllm_omni.model_executor.models.indextts2.text_processing_v2_5 as text_processing
monkeypatch.setattr(
text_processing,
"prepare_indextts25_text",
lambda *args, **kwargs: ([0, 58838, 1, 42, 0], 7),
)
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
talker.conditioning_policy = resolve_indextts_conditioning_policy("indextts2_5")
talker.model_path = "/model"
talker.config = SimpleNamespace(tokenizer_file="tokenizer.tiktoken")
token_ids, language_id = talker._tokenize_text(
"hello",
torch.device("cpu"),
lang="en",
)
assert token_ids.tolist() == [[0, 58838, 42, 1]]
assert language_id.tolist() == [7]
def test_native_fsdp_checkpoint_keys_are_normalized_at_every_nesting_level():
assert _normalize_indextts_checkpoint_key("_fsdp_wrapped_module.spk_emb_proj.weight") == "spk_emb_proj.weight"
assert (
_normalize_indextts_checkpoint_key("_fsdp_wrapped_module.mel_embedding._fsdp_wrapped_module.weight")
== "mel_embedding.weight"
)
def test_prefill_span_mismatch_fails_instead_of_padding_or_truncating():
for actual_span_len in (9, 11):
with pytest.raises(
ValueError,
match="model_type=indextts2_5.*lang=es.*text_token_count=6",
):
_require_exact_indextts_prefill_span(
actual_span_len=actual_span_len,
scheduled_span_len=10,
model_type="indextts2_5",
lang="es",
text_token_count=6,
)
def test_matching_prefill_span_is_accepted():
_require_exact_indextts_prefill_span(
actual_span_len=10,
scheduled_span_len=10,
model_type="indextts2_5",
lang="en",
text_token_count=6,
)
def _make_minimal_talker(
monkeypatch,
*,
model_type,
use_gpt_latent,
):
import vllm_omni.model_executor.models.indextts2.indextts2_talker as talker_module
seen = {}
def fake_parallel_lm_head(vocab_size, hidden_size, *, bias=False, **_kwargs):
seen["bias"] = bias
return torch.nn.Linear(hidden_size, vocab_size, bias=bias)
monkeypatch.setattr(talker_module, "ParallelLMHead", fake_parallel_lm_head)
monkeypatch.setattr(
talker_module,
"make_layers",
lambda *_args, **_kwargs: (
0,
1,
torch.nn.ModuleList([torch.nn.Linear(8, 8)]),
),
)
monkeypatch.setattr(
talker_module,
"get_pp_group",
lambda: SimpleNamespace(is_last_rank=lambda: True),
)
monkeypatch.setattr(talker_module, "get_speaker_cache", object)
monkeypatch.setattr(
talker_module,
"LogitsProcessor",
lambda *_args, **_kwargs: object(),
)
monkeypatch.setattr(
talker_module,
"ConformerEncoder",
lambda **_kwargs: torch.nn.Linear(1, 1),
)
monkeypatch.setattr(
talker_module,
"PerceiverResampler",
lambda *_args, **_kwargs: torch.nn.Linear(1, 1),
)
gpt = {
"model_dim": 8,
"layers": 0,
"heads": 1,
"max_mel_tokens": 4,
"max_text_tokens": 4,
"number_mel_codes": 16,
"start_mel_token": 14,
"stop_mel_token": 15,
"number_text_tokens": 32,
"condition_num_latent": 1,
"condition_module": {},
"emo_condition_module": {},
}
config = SimpleNamespace(
gpt=gpt,
model_type=model_type,
use_gpt_latent=use_gpt_latent,
gpt_checkpoint="gpt.pth",
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(model="/model", hf_config=config),
cache_config=None,
quant_config=None,
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
)
return (
IndexTTS2TalkerForConditionalGeneration(vllm_config=vllm_config),
seen,
)
@pytest.mark.parametrize(
("model_type", "use_gpt_latent"),
[
("indextts2", True),
("indextts2_5", False),
],
)
def test_mel_head_preserves_official_checkpoint_bias_and_hidden_payload_policy(
monkeypatch,
model_type,
use_gpt_latent,
):
talker, seen = _make_minimal_talker(
monkeypatch,
model_type=model_type,
use_gpt_latent=use_gpt_latent,
)
assert seen["bias"] is True
assert talker.mel_head.bias is not None
assert talker.omni_pooler_payload_include_hidden is use_gpt_latent
assert talker.omni_payload_at_request_end is (model_type == "indextts2_5")
def test_v2_load_weights_loads_exact_mel_head_bias_parameter(monkeypatch):
import vllm_omni.model_executor.models.indextts2.indextts2_talker as talker_module
talker, _ = _make_minimal_talker(
monkeypatch,
model_type="indextts2",
use_gpt_latent=True,
)
checkpoint = {
name: parameter.detach().clone() for name, parameter in talker.named_parameters(remove_duplicate=False)
}
expected_bias = torch.arange(
talker.number_mel_codes,
dtype=talker.mel_head.bias.dtype,
)
checkpoint["mel_head.bias"] = expected_bias
with torch.no_grad():
talker.mel_head.bias.zero_()
monkeypatch.setattr(
talker_module,
"resolve_model_file",
lambda *_args, **_kwargs: "/model/gpt.pth",
)
monkeypatch.setattr(
talker_module.torch,
"load",
lambda *_args, **_kwargs: checkpoint,
)
monkeypatch.setattr(
talker_module,
"is_pp_missing_parameter",
lambda *_args, **_kwargs: False,
)
monkeypatch.setattr(
talker,
"_warmup_external_models",
lambda _device: None,
)
loaded = talker.load_weights([])
assert "mel_head.bias" in loaded
torch.testing.assert_close(talker.mel_head.bias, expected_bias)
def test_compute_logits_passes_mel_head_bias_to_logits_processor():
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
torch.nn.Module.__init__(talker)
talker.mel_head = torch.nn.Linear(3, 4, bias=True)
hidden_states = torch.zeros(1, 3)
seen = {}
def fake_logits_processor(head, hidden, **kwargs):
seen["head"] = head
seen["hidden"] = hidden
seen.update(kwargs)
return hidden
talker.logits_processor = fake_logits_processor
result = talker.compute_logits(hidden_states)
assert result is hidden_states
assert seen["head"] is talker.mel_head
assert seen["hidden"] is hidden_states
assert seen["embedding_bias"] is talker.mel_head.bias
def _make_cached_v25_prefill_talker(monkeypatch):
talker, _ = _make_minimal_talker(
monkeypatch,
model_type="indextts2_5",
use_gpt_latent=False,
)
talker._speaker_cache = SimpleNamespace(
make_cache_key=lambda *_args: "cached-speaker",
get=lambda _key: {
"S_ref": torch.ones(1, 3, 8),
"ref_mel": torch.ones(1, 80, 3),
"style": torch.ones(1, 192),
"spk_cond_emb": torch.ones(1, 3, 8),
},
)
monkeypatch.setattr(
talker,
"_compute_emotion_vector",
lambda **_kwargs: torch.zeros(1, talker.model_dim),
)
monkeypatch.setattr(
talker,
"_tokenize_text",
lambda *_args, **_kwargs: (torch.tensor([[1]]), None),
)
return talker
def test_prefill_preserves_duration_factor_in_metadata(monkeypatch):
talker = _make_cached_v25_prefill_talker(monkeypatch)
_, _, info_update = talker._preprocess_prefill(
torch.zeros(5, dtype=torch.long),
None,
{
"text": ["hello"],
"voice": ["/speaker.wav"],
"voice_name": ["cached"],
"duration_factor": [0.5],
},
span_len=5,
)
assert info_update["meta"]["duration_factor"] == 0.5
@pytest.mark.parametrize("duration_factor", [0.49, 2.01])
def test_prefill_rejects_duration_factor_outside_supported_range(monkeypatch, duration_factor):
talker = _make_cached_v25_prefill_talker(monkeypatch)
with pytest.raises(
ValueError,
match="^IndexTTS 2.5 duration_factor must be between 0.5 and 2.0$",
):
talker._preprocess_prefill(
torch.zeros(5, dtype=torch.long),
None,
{
"text": ["hello"],
"voice": ["/speaker.wav"],
"voice_name": ["cached"],
"duration_factor": [duration_factor],
},
span_len=5,
)
def test_no_latent_sampled_token_payload_emits_only_first_step_metadata():
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
talker.use_gpt_latent = False
hidden = torch.zeros(1, 8)
info = {
"codes": {"mel": torch.tensor([17])},
"meta": {
"mel_code_count": 1,
"S_ref": torch.ones(1, 4),
"ref_mel": torch.ones(1, 80, 3),
"style": torch.ones(1, 192),
"duration_factor": 0.5,
},
}
first_decode_output = talker.make_omni_output(
hidden,
model_intermediate_buffer=[info],
)
assert "codes" not in first_decode_output.multimodal_outputs
assert "hidden_states" not in first_decode_output.multimodal_outputs
assert first_decode_output.multimodal_outputs["meta"][0]["S_ref"] is info["meta"]["S_ref"]
assert first_decode_output.multimodal_outputs["meta"][0]["duration_factor"] == 0.5
later = talker.make_omni_output(
hidden,
model_intermediate_buffer=[
{
"codes": {"mel": torch.tensor([23])},
"meta": {"mel_code_count": 2},
}
],
)
assert later.multimodal_outputs == {"meta": [{"use_gpt_latent": False}]}
def test_first_decode_payload_defaults_missing_duration_factor_to_one():
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
talker.use_gpt_latent = False
hidden = torch.zeros(1, 8)
info = {
"codes": {"mel": torch.tensor([17])},
"meta": {
"mel_code_count": 1,
"S_ref": torch.ones(1, 4),
"ref_mel": torch.ones(1, 80, 3),
"style": torch.ones(1, 192),
},
}
output = talker.make_omni_output(
hidden,
model_intermediate_buffer=[info],
)
assert output.multimodal_outputs["meta"][0]["duration_factor"] == 1.0
def test_latent_sampled_token_payload_keeps_aligned_gpu_deltas():
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
talker.use_gpt_latent = True
hidden = torch.zeros(1, 8)
info = {
"codes": {"mel": torch.tensor([17])},
"meta": {
"mel_code_count": 1,
"latent_acc": torch.arange(8, dtype=torch.float32).reshape(1, 8),
"duration_factor": 1.0,
},
}
output = talker.make_omni_output(
hidden,
model_intermediate_buffer=[info],
)
assert output.multimodal_outputs["codes"]["mel"][0].tolist() == [[17]]
assert output.multimodal_outputs["hidden_states"]["latent"][0].tolist() == [list(range(8))]
def test_v25_default_emotion_reuses_speaker_w2v_features(monkeypatch):
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
speaker_features = torch.zeros(1, 4, 1024)
expected = torch.ones(1, 1280)
seen = {}
def fake_merge(spk_cond_emb, emo_cond_emb, alpha):
seen["spk"] = spk_cond_emb
seen["emo"] = emo_cond_emb
seen["alpha"] = alpha
return expected
monkeypatch.setattr(talker, "_merge_emovec", fake_merge)
result = talker._compute_emotion_vector(
wav_16k=None,
speaker_audio_path="/speaker.wav",
emo_audio_path=None,
emo_voice_name=None,
main_text="hello",
use_emo_text=False,
emo_text=None,
emo_vector=None,
emo_alpha=0.25,
use_random=False,
style=torch.zeros(1, 192),
spk_cond_emb=speaker_features,
w2v_model=None,
w2v_proc=None,
device=torch.device("cpu"),
)
assert result is expected
assert seen == {
"spk": speaker_features,
"emo": speaker_features,
"alpha": 1.0,
}
@@ -0,0 +1,244 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import builtins
import pytest
from vllm_omni.model_executor.models.indextts2 import text_processing_v2_5
from vllm_omni.model_executor.models.indextts2.text_processing_v2_5 import (
apply_pronunciation_annotations,
clean_indextts25_text,
prepare_indextts25_text,
)
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
INDEXTTS25_SPECIAL_TOKENS,
INDEXTTS25_VOCAB_SIZE,
LANGUAGE_DICT,
lang_to_token,
normalize_language_code,
resolve_indextts25_tokenizer_file,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_special_token_layout_matches_official_checkpoint_vocab():
assert len(INDEXTTS25_SPECIAL_TOKENS) == 1673
assert len(set(INDEXTTS25_SPECIAL_TOKENS)) == 1673
assert 58836 + len(INDEXTTS25_SPECIAL_TOKENS) == INDEXTTS25_VOCAB_SIZE
assert INDEXTTS25_VOCAB_SIZE == 60509
def test_pronunciation_annotations_select_english_and_chinese_markers():
assert apply_pronunciation_annotations("<going|G OW1 . IH0 NG>") == (
"<|SPECIAL_TOKEN_1|>G OW1 . IH0 NG<|SPECIAL_TOKEN_1|>"
)
assert apply_pronunciation_annotations("<行|xing2>") == ("<|SPECIAL_TOKEN_2|>XING2<|SPECIAL_TOKEN_2|>")
def test_prepare_text_applies_language_prefix_case_and_annotation(monkeypatch):
captured = {}
def fake_encode(text, *, model_dir, tokenizer_file):
captured["text"] = text
captured["model_dir"] = model_dir
captured["tokenizer_file"] = tokenizer_file
return [7, 8, 9]
monkeypatch.setattr(text_processing_v2_5, "encode_indextts25_text", fake_encode)
token_ids, lang_id = prepare_indextts25_text(
"HELLO <行|xing2>",
lang="zh",
model_dir="/model",
text_normalization=False,
)
assert token_ids == [7, 8, 9]
assert lang_id == LANGUAGE_DICT["zh"]
assert captured == {
"text": "<|zh|> hello <|SPECIAL_TOKEN_2|>XING2<|SPECIAL_TOKEN_2|>",
"model_dir": "/model",
"tokenizer_file": "multilingual_zh_ja_yue_char_del.tiktoken",
}
def test_mandarin_alias_is_an_intentional_vllm_omni_convenience():
assert normalize_language_code("Mandarin") == "zh"
assert lang_to_token("mandarin") == LANGUAGE_DICT["zh"]
def test_invalid_language_does_not_fall_back_to_common():
with pytest.raises(ValueError, match="Unsupported IndexTTS 2.5 language"):
lang_to_token("xx-invalid")
def test_zhen_uses_mixed_normalization():
calls = []
def fake_mixed_normalizer(text):
calls.append(text)
return "MIXED"
original = text_processing_v2_5._normalize_zh_or_en
text_processing_v2_5._normalize_zh_or_en = fake_mixed_normalizer
try:
result = text_processing_v2_5._normalize_with_official_backend(
"中文 and English",
"zhen",
)
finally:
text_processing_v2_5._normalize_zh_or_en = original
assert calls == ["中文 and English"]
assert result == "MIXED"
def test_spanish_normalization_does_not_hide_broken_dependencies(monkeypatch):
original_import = builtins.__import__
def controlled_import(name, *args, **kwargs):
if name == "nemo_text_processing.text_normalization.normalize":
raise RuntimeError("broken NeMo grammar")
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", controlled_import)
with pytest.raises(RuntimeError, match="broken NeMo grammar"):
text_processing_v2_5._normalize_with_official_backend("hola", "es")
def test_zhen_keeps_literal_prefix_and_uses_common_embedding(monkeypatch):
captured = {}
def fake_encode(text, *, model_dir, tokenizer_file):
captured["text"] = text
return [7, 8, 9]
monkeypatch.setattr(text_processing_v2_5, "encode_indextts25_text", fake_encode)
token_ids, lang_id = prepare_indextts25_text(
"HELLO 中文",
lang="zhen",
model_dir="/model",
text_normalization=False,
)
assert token_ids == [7, 8, 9]
assert captured["text"] == "<|zhen|> hello 中文"
assert lang_id == LANGUAGE_DICT["common"]
assert "<|zhen|>" not in INDEXTTS25_SPECIAL_TOKENS
assert "<|common|>" not in INDEXTTS25_SPECIAL_TOKENS
@pytest.mark.parametrize(
("text", "expected"),
[
(",,,", "…"),
("测试,,,结束", "测试…结束"),
],
)
def test_repeated_full_width_commas_use_longest_first_cleanup(text, expected):
assert clean_indextts25_text(text) == expected
def test_default_text_normalizer_is_shared_by_prompt_and_talker(monkeypatch):
calls = []
def fake_normalizer(text, lang):
calls.append((text, lang))
return "twenty five percent"
monkeypatch.setattr(
text_processing_v2_5,
"_normalize_with_official_backend",
fake_normalizer,
)
result = text_processing_v2_5.normalize_indextts25_text(
"25%",
lang="en",
text_normalization=True,
)
assert calls == [("25%", "en")]
assert result == "twenty five percent"
def test_mixed_text_uses_official_content_routing_and_protection(monkeypatch):
calls = []
class FakeNormalizer:
def __init__(self, lang):
self.lang = lang
def normalize(self, text):
calls.append((self.lang, text))
return text.replace("<H>", " <H> ").replace("5", "five")
monkeypatch.setattr(
text_processing_v2_5,
"_load_wetext_normalizer",
lambda lang: FakeNormalizer(lang),
)
result = text_processing_v2_5._normalize_with_official_backend(
"GPT-5-nano 中文 xuan4",
"en",
)
assert calls[0][0] == "zh"
assert result == "GPT-five-nano 中文 XVAN4"
def test_tokenizer_resolves_native_checkpoints_subdirectory(tmp_path):
tokenizer = tmp_path / "checkpoints" / "multilingual_zh_ja_yue_char_del.tiktoken"
tokenizer.parent.mkdir()
tokenizer.touch()
assert resolve_indextts25_tokenizer_file(str(tmp_path)) == str(tokenizer)
@pytest.mark.parametrize(
("lang", "text", "expected"),
[
(
"es",
"En la actualidad se continúa utilizando una corona abierta.",
"<|es|> EN LA ACTUALIDAD SE CONTINÚA UTILIZANDO UNA CORONA ABIERTA.",
),
(
"ar",
"فكلُّها خياراتٌ اتَّخذتَها بنفسِك.",
"<|ar|> فكلُّها خياراتٌ اتَّخذتَها بنفسِك.",
),
],
)
def test_official_spanish_and_arabic_cases_preserve_expected_text_contract(
monkeypatch,
lang,
text,
expected,
):
captured = {}
def fake_encode(value, *, model_dir, tokenizer_file):
captured["text"] = value
return [1]
monkeypatch.setattr(
text_processing_v2_5,
"encode_indextts25_text",
fake_encode,
)
_, lang_id = prepare_indextts25_text(
text,
lang=lang,
model_dir="/model",
text_normalization=False,
)
assert captured["text"] == expected
assert lang_id == LANGUAGE_DICT[lang]
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.indextts2.indextts2_s2mel_decoder import (
IndexTTS2S2MelDecoder,
)
from vllm_omni.model_executor.models.indextts2.s2mel.modules.wavenet import WN
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_wavenet() -> WN:
torch.manual_seed(0)
return WN(
hidden_channels=2,
kernel_size=5,
dilation_rate=1,
n_layers=8,
gin_channels=2,
p_dropout=0.0,
causal=False,
).eval()
def _sequence_mask(lengths: torch.Tensor, max_length: int) -> torch.Tensor:
positions = torch.arange(max_length, device=lengths.device)
return (positions.unsqueeze(0) < lengths.unsqueeze(1)).unsqueeze(1)
def _singleton_outputs(
wavenet: WN,
inputs: torch.Tensor,
lengths: list[int],
conditioning: torch.Tensor,
) -> list[torch.Tensor]:
# This is the independent oracle: every request ends at the physical tensor
# boundary and therefore uses SConv1d's original reflect-padding behavior.
wavenet._assume_full_mask = True
return [
wavenet(
inputs[row : row + 1, :, :length],
torch.ones(1, 1, length, dtype=torch.bool),
g=conditioning[row : row + 1],
)
for row, length in enumerate(lengths)
]
def test_ragged_wavenet_matches_original_singleton_reflect_boundaries():
wavenet = _make_wavenet()
lengths = [6, 5, 4, 3]
inputs = torch.randn(len(lengths), 2, max(lengths))
conditioning = torch.randn(len(lengths), 2, 1)
expected = _singleton_outputs(wavenet, inputs, lengths, conditioning)
mask = _sequence_mask(torch.tensor(lengths), inputs.shape[-1])
wavenet._assume_full_mask = False
# Keep deliberately non-zero invalid tails. The first WaveNet layer sees
# transformer output before the recurrent mask has cleared those values.
actual = wavenet(inputs, mask, g=conditioning)
for row, (length, singleton) in enumerate(zip(lengths, expected)):
torch.testing.assert_close(
actual[row : row + 1, :, :length],
singleton,
rtol=1e-5,
atol=1e-6,
)
def test_wavenet_exposes_minimum_safe_ragged_length():
# kernel=5, dilation=1 has two right-reflect ghost positions, so a
# logical row must contain at least three frames before it can use the
# vectorized ragged path. Shorter rows stay on original singleton SConv1d.
assert _make_wavenet().ragged_min_length == 3
def test_cfm_grouping_isolates_rows_too_short_for_logical_reflect():
groups = IndexTTS2S2MelDecoder._cfm_group_indices(
sorted_indices=[0, 1, 2, 3, 4],
total_lengths=[0, 1, 2, 3, 6],
max_group_size=2,
min_ragged_length=3,
)
assert groups == [[0], [1], [2], [3, 4]]
def test_ragged_wavenet_preserves_cfg_cond_uncond_row_order():
wavenet = _make_wavenet()
base_lengths = [3, 6]
conditional = torch.randn(2, 2, 6)
unconditional = torch.randn(2, 2, 6)
cfg_inputs = torch.cat([conditional, unconditional], dim=0)
cfg_lengths = base_lengths + base_lengths
cfg_conditioning = torch.randn(4, 2, 1)
expected = _singleton_outputs(
wavenet,
cfg_inputs,
cfg_lengths,
cfg_conditioning,
)
mask = _sequence_mask(torch.tensor(cfg_lengths), cfg_inputs.shape[-1])
wavenet._assume_full_mask = False
actual = wavenet(cfg_inputs, mask, g=cfg_conditioning)
for row, (length, singleton) in enumerate(zip(cfg_lengths, expected)):
torch.testing.assert_close(
actual[row : row + 1, :, :length],
singleton,
rtol=1e-5,
atol=1e-6,
)
def test_full_mask_wavenet_path_is_bit_exact_with_original_path():
wavenet = _make_wavenet()
inputs = torch.randn(3, 2, 6)
mask = torch.ones(3, 1, 6, dtype=torch.bool)
conditioning = torch.randn(3, 2, 1)
if hasattr(wavenet, "_assume_full_mask"):
del wavenet._assume_full_mask
expected = wavenet(inputs, mask, g=conditioning)
wavenet._assume_full_mask = True
actual = wavenet(inputs, mask, g=conditioning)
assert torch.equal(actual, expected)
@pytest.mark.skipif(not hasattr(torch, "compile"), reason="torch.compile unavailable")
def test_ragged_wavenet_is_one_dynamic_fullgraph():
wavenet = _make_wavenet()
lengths = [3, 6]
inputs = torch.randn(2, 2, 6)
conditioning = torch.randn(2, 2, 1)
mask = _sequence_mask(torch.tensor(lengths), inputs.shape[-1])
expected = _singleton_outputs(wavenet, inputs, lengths, conditioning)
wavenet._assume_full_mask = False
compiled = torch.compile(
wavenet,
backend="eager",
fullgraph=True,
dynamic=True,
)
actual = compiled(inputs, mask, g=conditioning)
for row, (length, singleton) in enumerate(zip(lengths, expected)):
torch.testing.assert_close(
actual[row : row + 1, :, :length],
singleton,
rtol=1e-5,
atol=1e-6,
)
def test_decoder_full_mask_switch_also_controls_wavenet_boundary_path():
decoder = object.__new__(IndexTTS2S2MelDecoder)
attention_layers = [SimpleNamespace(attention=SimpleNamespace()) for _ in range(2)]
estimator = SimpleNamespace(
transformer=SimpleNamespace(layers=attention_layers),
wavenet=SimpleNamespace(),
)
decoder._set_dit_full_mask_fast_path(estimator, enabled=False)
assert all(layer.attention._assume_full_mask is False for layer in attention_layers)
assert estimator.wavenet._assume_full_mask is False
decoder._set_dit_full_mask_fast_path(estimator, enabled=True)
assert all(layer.attention._assume_full_mask is True for layer in attention_layers)
assert estimator.wavenet._assume_full_mask is True
def test_decoder_full_mask_switch_unwraps_compiled_estimator():
decoder = object.__new__(IndexTTS2S2MelDecoder)
attention_layers = [SimpleNamespace(attention=SimpleNamespace()) for _ in range(2)]
eager_estimator = SimpleNamespace(
transformer=SimpleNamespace(layers=attention_layers),
wavenet=SimpleNamespace(),
)
compiled_estimator = SimpleNamespace(_orig_mod=eager_estimator)
decoder._set_dit_full_mask_fast_path(compiled_estimator, enabled=False)
assert all(layer.attention._assume_full_mask is False for layer in attention_layers)
assert eager_estimator.wavenet._assume_full_mask is False