[New Model] Support IndexTTS 2.5 (#5957)
Signed-off-by: Sysy <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
1ff2753469
commit
58007c0084
77 files changed
+7123
-519
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.model_executor.models.indextts2.configuration_indextts2 import (
|
||||
IndexTTS2Config,
|
||||
IndexTTS25Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.indextts2.pipeline import INDEXTTS25_PIPELINE
|
||||
|
||||
|
||||
def test_indextts25_defaults_are_distinct_from_v2():
|
||||
v2 = IndexTTS2Config()
|
||||
v25 = IndexTTS25Config()
|
||||
|
||||
assert v2.model_type == "indextts2"
|
||||
assert v2.gpt["number_text_tokens"] == 12000
|
||||
assert v2.gpt["condition_num_latent"] == 32
|
||||
assert v2.use_gpt_latent is True
|
||||
assert v2.semantic_codec_type == "repcodec"
|
||||
assert v2.vocab_size == v2.gpt["number_mel_codes"]
|
||||
|
||||
assert v25.model_type == "indextts2_5"
|
||||
assert v25.gpt["number_text_tokens"] == 60509
|
||||
assert v25.gpt["condition_num_latent"] == 32
|
||||
assert v25.use_gpt_latent is False
|
||||
assert v25.semantic_codec_type == "enhanced"
|
||||
assert v25.semantic_codec_checkpoint == "codec.pth"
|
||||
assert v25.tokenizer_file == "multilingual_zh_ja_yue_char_del.tiktoken"
|
||||
assert v25.vocab_size == v25.gpt["number_mel_codes"]
|
||||
|
||||
|
||||
def test_indextts25_explicit_overrides_win_over_defaults():
|
||||
config = IndexTTS25Config(
|
||||
use_gpt_latent=True,
|
||||
gpt={
|
||||
"number_text_tokens": 70000,
|
||||
"condition_num_latent": 2,
|
||||
},
|
||||
vocab_size=9000,
|
||||
)
|
||||
|
||||
assert config.use_gpt_latent is True
|
||||
assert config.gpt["number_text_tokens"] == 70000
|
||||
assert config.gpt["condition_num_latent"] == 2
|
||||
assert config.gpt["model_dim"] == config.hidden_size
|
||||
assert config.vocab_size == 9000
|
||||
|
||||
|
||||
def test_indextts25_pipeline_is_registered_with_two_distinct_stages():
|
||||
assert OMNI_PIPELINES["indextts2_5"] is INDEXTTS25_PIPELINE
|
||||
assert INDEXTTS25_PIPELINE.default_deploy_config_name == "indextts2_5.yaml"
|
||||
assert [stage.model_stage for stage in INDEXTTS25_PIPELINE.stages] == [
|
||||
"indextts2_5_talker",
|
||||
"indextts2_5_s2mel_decoder",
|
||||
]
|
||||
assert [stage.model_arch for stage in INDEXTTS25_PIPELINE.stages] == [
|
||||
None,
|
||||
"IndexTTS25S2MelDecoder",
|
||||
]
|
||||
assert all("tokenizer" not in stage.extras for stage in INDEXTTS25_PIPELINE.stages)
|
||||
|
||||
|
||||
def test_indextts25_default_deploy_selects_validated_triton_backend():
|
||||
deploy_path = Path(__file__).parents[4] / "vllm_omni" / "deploy" / "indextts2_5.yaml"
|
||||
config = yaml.safe_load(deploy_path.read_text(encoding="utf-8"))
|
||||
stage0 = next(stage for stage in config["stages"] if stage["stage_id"] == 0)
|
||||
|
||||
assert stage0["attention_backend"] == "TRITON_ATTN"
|
||||
assert stage0["enable_chunked_prefill"] is False
|
||||
|
||||
|
||||
def test_indextts25_deploy_configs_keep_latent_policy_in_sync():
|
||||
deploy_dir = Path(__file__).parents[4] / "vllm_omni" / "deploy"
|
||||
expected = {"indextts2_5.yaml": False}
|
||||
for filename, use_gpt_latent in expected.items():
|
||||
with open(deploy_dir / filename, encoding="utf-8") as deploy_file:
|
||||
config = yaml.safe_load(deploy_file)
|
||||
assert config["pipeline"] == "indextts2_5"
|
||||
assert config["async_chunk"] is False
|
||||
assert all("tokenizer" not in stage for stage in config["stages"])
|
||||
assert [stage["hf_overrides"]["use_gpt_latent"] for stage in config["stages"]] == [
|
||||
use_gpt_latent,
|
||||
use_gpt_latent,
|
||||
]
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2.dit_cuda_graph import (
|
||||
CUDAGraphDiTRunner,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _CopySpy:
|
||||
def __init__(self):
|
||||
self.sources = []
|
||||
|
||||
def copy_(self, source):
|
||||
self.sources.append(source)
|
||||
return self
|
||||
|
||||
|
||||
class _GraphSpy:
|
||||
def __init__(self):
|
||||
self.replays = 0
|
||||
|
||||
def replay(self):
|
||||
self.replays += 1
|
||||
|
||||
|
||||
def test_replay_only_updates_dynamic_inputs_and_reuses_static_output():
|
||||
runner = CUDAGraphDiTRunner(transformer=object())
|
||||
graph = _GraphSpy()
|
||||
static_x = _CopySpy()
|
||||
static_c = _CopySpy()
|
||||
static_pos = _CopySpy()
|
||||
static_mask = _CopySpy()
|
||||
static_out = object()
|
||||
entry = SimpleNamespace(
|
||||
graph=graph,
|
||||
static_x=static_x,
|
||||
static_c=static_c,
|
||||
static_pos=static_pos,
|
||||
static_mask=static_mask,
|
||||
static_out=static_out,
|
||||
)
|
||||
x_in = object()
|
||||
c = object()
|
||||
|
||||
output = runner._replay(entry, x_in=x_in, c=c)
|
||||
|
||||
assert static_x.sources == [x_in]
|
||||
assert static_c.sources == [c]
|
||||
assert static_pos.sources == []
|
||||
assert static_mask.sources == []
|
||||
assert graph.replays == 1
|
||||
assert output is static_out
|
||||
|
||||
|
||||
def test_graph_stats_start_empty_and_remain_bounded():
|
||||
runner = CUDAGraphDiTRunner(transformer=object(), max_graphs=4)
|
||||
|
||||
stats = runner.stats_snapshot()
|
||||
stats["hits"] = 99
|
||||
|
||||
assert runner.stats_snapshot() == {
|
||||
"calls": 0,
|
||||
"hits": 0,
|
||||
"captures": 0,
|
||||
"capture_failures": 0,
|
||||
"evictions": 0,
|
||||
"eager": 0,
|
||||
"cache_size": 0,
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2 import codec as codec_module
|
||||
from vllm_omni.model_executor.models.indextts2 import preprocess_utils
|
||||
from vllm_omni.model_executor.models.indextts2.codec import EnhancedCodec
|
||||
|
||||
|
||||
def test_enhanced_codec_decode_restores_hidden_size_and_doubles_time():
|
||||
codec = EnhancedCodec(
|
||||
codebook_size=8,
|
||||
hidden_size=4,
|
||||
codebook_dim=2,
|
||||
vocos_dim=4,
|
||||
vocos_intermediate_dim=8,
|
||||
vocos_num_layers=1,
|
||||
downsample_scale=2,
|
||||
).eval()
|
||||
|
||||
decoded = codec.decode(torch.tensor([[0, 1, 2]]))
|
||||
|
||||
assert decoded.shape == (1, 6, 4)
|
||||
|
||||
|
||||
def test_enhanced_codec_inference_module_does_not_allocate_unused_encoder():
|
||||
codec = EnhancedCodec(
|
||||
codebook_size=8,
|
||||
hidden_size=4,
|
||||
codebook_dim=2,
|
||||
vocos_dim=4,
|
||||
vocos_intermediate_dim=8,
|
||||
vocos_num_layers=1,
|
||||
)
|
||||
|
||||
parameter_names = set(dict(codec.named_parameters()))
|
||||
assert not any(name.startswith("encoder.") for name in parameter_names)
|
||||
assert not any(name.startswith("down.") for name in parameter_names)
|
||||
assert any(name.startswith("decoder.") for name in parameter_names)
|
||||
assert any(name.startswith("quantizer.") for name in parameter_names)
|
||||
|
||||
|
||||
def test_semantic_codec_loader_selects_enhanced_checkpoint(tmp_path, monkeypatch):
|
||||
checkpoint = tmp_path / "codec.pth"
|
||||
checkpoint.touch()
|
||||
seen = {}
|
||||
|
||||
class FakeCodec:
|
||||
def load_checkpoint(self, path):
|
||||
seen["path"] = path
|
||||
|
||||
def to(self, **kwargs):
|
||||
seen["to"] = kwargs
|
||||
return self
|
||||
|
||||
def eval(self):
|
||||
return self
|
||||
|
||||
def parameters(self):
|
||||
return ()
|
||||
|
||||
monkeypatch.setattr(codec_module, "EnhancedCodec", lambda **kwargs: FakeCodec())
|
||||
preprocess_utils._semantic_codec_cache.clear()
|
||||
|
||||
codec = preprocess_utils.load_semantic_codec(
|
||||
str(tmp_path),
|
||||
{"codebook_size": 8192},
|
||||
torch.device("cpu"),
|
||||
codec_type="enhanced",
|
||||
checkpoint_name="codec.pth",
|
||||
)
|
||||
|
||||
assert isinstance(codec, FakeCodec)
|
||||
assert seen["path"] == str(checkpoint)
|
||||
assert seen["to"] == {"device": torch.device("cpu"), "dtype": torch.float32}
|
||||
|
||||
|
||||
def test_model_asset_resolver_supports_native_checkpoints_layout(tmp_path):
|
||||
checkpoint = tmp_path / "checkpoints" / "codec.pth"
|
||||
checkpoint.parent.mkdir()
|
||||
checkpoint.touch()
|
||||
|
||||
assert preprocess_utils.resolve_model_file(str(tmp_path), "codec.pth") == str(checkpoint)
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2 import prompt_utils
|
||||
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
|
||||
INDEXTTS25_TOKENIZER_FILE,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _load_indextts2_offline_example():
|
||||
example_path = (
|
||||
Path(__file__).resolve().parents[4]
|
||||
/ "examples"
|
||||
/ "offline_inference"
|
||||
/ "text_to_speech"
|
||||
/ "indextts2"
|
||||
/ "end2end.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"indextts2_offline_example_test",
|
||||
example_path,
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_offline_main_uses_resolved_stage0_tokenizer_file(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
example = _load_indextts2_offline_example()
|
||||
stage0 = SimpleNamespace(engine_args=SimpleNamespace(hf_overrides={"tokenizer_file": "custom-tokenizer.tiktoken"}))
|
||||
fake_omni = SimpleNamespace(
|
||||
stage_configs=[stage0],
|
||||
generate=lambda *args, **kwargs: [],
|
||||
)
|
||||
captured = {}
|
||||
|
||||
monkeypatch.setattr(example, "Omni", lambda **kwargs: fake_omni)
|
||||
monkeypatch.setattr(example, "SamplingParams", lambda **kwargs: kwargs)
|
||||
monkeypatch.setattr(
|
||||
example,
|
||||
"build_request",
|
||||
lambda **kwargs: captured.update(kwargs) or {"additional_information": {}},
|
||||
)
|
||||
|
||||
example.main(
|
||||
SimpleNamespace(
|
||||
model="/model",
|
||||
model_version="2.5",
|
||||
deploy_config="/deploy/custom.yaml",
|
||||
use_gpt_latent=False,
|
||||
stage_init_timeout=300,
|
||||
seed=42,
|
||||
output_dir=str(tmp_path),
|
||||
text="hello",
|
||||
ref_audio=None,
|
||||
emo_audio=None,
|
||||
emo_text=None,
|
||||
emo_vector=None,
|
||||
emo_alpha=None,
|
||||
use_emo_text=False,
|
||||
use_random=False,
|
||||
lang="en",
|
||||
text_normalization=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
|
||||
|
||||
|
||||
def test_indextts25_prompt_len_uses_three_conditioning_tokens(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_prepare(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return [11, 12, 13, 14], 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"prepare_indextts25_text",
|
||||
fake_prepare,
|
||||
)
|
||||
|
||||
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
|
||||
"/model",
|
||||
"hello",
|
||||
model_type="indextts2_5",
|
||||
lang="en",
|
||||
text_normalization=False,
|
||||
)
|
||||
|
||||
# Official prepare_gpt_inputs wraps the language-prefixed tokenizer IDs
|
||||
# with start_text=0 and stop_text=1 before appending start_mel.
|
||||
assert prompt_len == 3 + (4 + 2) + 1
|
||||
assert captured["text_normalization"] is False
|
||||
assert captured["tokenizer_file"] == INDEXTTS25_TOKENIZER_FILE
|
||||
|
||||
|
||||
def test_indextts25_prompt_len_uses_custom_tokenizer_file(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_prepare(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return [11, 12], 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"prepare_indextts25_text",
|
||||
fake_prepare,
|
||||
)
|
||||
|
||||
prompt_utils.estimate_indextts2_prefill_prompt_len(
|
||||
"/model",
|
||||
"hello",
|
||||
model_type="indextts2_5",
|
||||
tokenizer_file="custom-tokenizer.tiktoken",
|
||||
)
|
||||
|
||||
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
|
||||
|
||||
|
||||
def test_indextts25_prompt_len_filters_existing_text_wrapper_ids(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"prepare_indextts25_text",
|
||||
lambda *args, **kwargs: ([0, 58838, 1, 42, 0], 0),
|
||||
)
|
||||
|
||||
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
|
||||
"/model",
|
||||
"hello",
|
||||
model_type="indextts2_5",
|
||||
lang="en",
|
||||
)
|
||||
|
||||
assert prompt_len == 3 + (2 + 2) + 1
|
||||
|
||||
|
||||
def test_indextts2_prompt_len_ignores_v25_tokenizer_file(monkeypatch):
|
||||
class FakeTokenizer:
|
||||
def encode(self, text, *, add_special_tokens):
|
||||
assert text == "hello"
|
||||
assert add_special_tokens is False
|
||||
return [11, 12]
|
||||
|
||||
def fail_prepare(*args, **kwargs):
|
||||
raise AssertionError("IndexTTS 2 must not use the IndexTTS 2.5 tokenizer")
|
||||
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"_get_text_tokenizer",
|
||||
lambda model_id_or_path: FakeTokenizer(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"prepare_indextts25_text",
|
||||
fail_prepare,
|
||||
)
|
||||
|
||||
prompt_len = prompt_utils.estimate_indextts2_prefill_prompt_len(
|
||||
"/model",
|
||||
"hello",
|
||||
model_type="indextts2",
|
||||
tokenizer_file="custom.tiktoken",
|
||||
)
|
||||
|
||||
assert prompt_len == 34 + (2 + 2) + 1
|
||||
|
||||
|
||||
def test_indextts25_prompt_ids_preserve_placeholder_value(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_estimate(*args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return 10
|
||||
|
||||
monkeypatch.setattr(
|
||||
prompt_utils,
|
||||
"estimate_indextts2_prefill_prompt_len",
|
||||
fake_estimate,
|
||||
)
|
||||
|
||||
prompt_ids = prompt_utils.build_indextts2_prefill_prompt_ids(
|
||||
"/model",
|
||||
"hello",
|
||||
model_type="indextts2_5",
|
||||
lang="en",
|
||||
tokenizer_file="custom-tokenizer.tiktoken",
|
||||
placeholder_token_id=17,
|
||||
)
|
||||
|
||||
assert prompt_ids == [17] * 10
|
||||
assert captured["tokenizer_file"] == "custom-tokenizer.tiktoken"
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def test_indextts25_default_recipe_enables_validated_cfm_batching():
|
||||
recipe_path = Path(__file__).parents[4] / "vllm_omni" / "deploy" / "indextts2_5.yaml"
|
||||
recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8"))
|
||||
stage1 = next(stage for stage in recipe["stages"] if stage["stage_id"] == 1)
|
||||
|
||||
assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.stage_input_processors.indextts2 import (
|
||||
STOP_MEL_TOKEN,
|
||||
talker2s2mel_full_payload,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_no_latent_full_payload_uses_sampled_output_ids_and_strips_stop():
|
||||
request = SimpleNamespace(
|
||||
request_id="req-1",
|
||||
output_token_ids=[17, 23, STOP_MEL_TOKEN],
|
||||
sampling_params=SimpleNamespace(seed=42),
|
||||
)
|
||||
payload = {
|
||||
"meta": {
|
||||
"use_gpt_latent": False,
|
||||
"S_ref": torch.ones(1, 4),
|
||||
"ref_mel": torch.ones(1, 80, 3),
|
||||
"style": torch.ones(1, 192),
|
||||
}
|
||||
}
|
||||
|
||||
result = talker2s2mel_full_payload(None, payload, request)
|
||||
|
||||
assert result is not None
|
||||
assert result["mel_codes"].tolist() == [[17, 23]]
|
||||
assert result["code_lens"].tolist() == [2]
|
||||
assert result["use_gpt_latent"] is False
|
||||
assert result["seed"] == 42
|
||||
|
||||
|
||||
def test_no_latent_full_payload_rejects_legacy_payload_codes():
|
||||
request = SimpleNamespace(
|
||||
request_id="legacy",
|
||||
output_token_ids=[],
|
||||
sampling_params=SimpleNamespace(seed=None),
|
||||
)
|
||||
payload = {
|
||||
"codes": {"mel": torch.tensor([31, 37, STOP_MEL_TOKEN])},
|
||||
"meta": {"use_gpt_latent": False},
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="no completed output_token_ids"):
|
||||
talker2s2mel_full_payload(None, payload, request)
|
||||
@@ -0,0 +1,476 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.config import CompilationMode
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2.indextts2_talker import (
|
||||
IndexTTS2TalkerForConditionalGeneration,
|
||||
_normalize_indextts_checkpoint_key,
|
||||
_require_exact_indextts_prefill_span,
|
||||
resolve_indextts_conditioning_policy,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_v25_conditioning_policy():
|
||||
policy = resolve_indextts_conditioning_policy("indextts2_5")
|
||||
|
||||
assert policy.prefix_tokens == 3
|
||||
assert policy.uses_conformer_perceiver is False
|
||||
assert policy.uses_campplus_projection is True
|
||||
assert policy.uses_language_embedding is True
|
||||
|
||||
|
||||
def test_v2_conditioning_policy_is_unchanged():
|
||||
policy = resolve_indextts_conditioning_policy("indextts2")
|
||||
|
||||
assert policy.prefix_tokens == 34
|
||||
assert policy.uses_conformer_perceiver is True
|
||||
assert policy.uses_campplus_projection is False
|
||||
assert policy.uses_language_embedding is False
|
||||
|
||||
|
||||
def test_v25_text_tokens_wrap_language_prefixed_ids_with_start_and_stop(
|
||||
monkeypatch,
|
||||
):
|
||||
import vllm_omni.model_executor.models.indextts2.text_processing_v2_5 as text_processing
|
||||
|
||||
monkeypatch.setattr(
|
||||
text_processing,
|
||||
"prepare_indextts25_text",
|
||||
lambda *args, **kwargs: ([58838, 42], 7),
|
||||
)
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
talker.conditioning_policy = resolve_indextts_conditioning_policy("indextts2_5")
|
||||
talker.model_path = "/model"
|
||||
talker.config = SimpleNamespace(tokenizer_file="tokenizer.tiktoken")
|
||||
|
||||
token_ids, language_id = talker._tokenize_text(
|
||||
"hello",
|
||||
torch.device("cpu"),
|
||||
lang="ja",
|
||||
)
|
||||
|
||||
assert token_ids.tolist() == [[0, 58838, 42, 1]]
|
||||
assert language_id.tolist() == [7]
|
||||
|
||||
|
||||
def test_v25_text_tokens_filter_existing_start_and_stop_ids(monkeypatch):
|
||||
import vllm_omni.model_executor.models.indextts2.text_processing_v2_5 as text_processing
|
||||
|
||||
monkeypatch.setattr(
|
||||
text_processing,
|
||||
"prepare_indextts25_text",
|
||||
lambda *args, **kwargs: ([0, 58838, 1, 42, 0], 7),
|
||||
)
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
talker.conditioning_policy = resolve_indextts_conditioning_policy("indextts2_5")
|
||||
talker.model_path = "/model"
|
||||
talker.config = SimpleNamespace(tokenizer_file="tokenizer.tiktoken")
|
||||
|
||||
token_ids, language_id = talker._tokenize_text(
|
||||
"hello",
|
||||
torch.device("cpu"),
|
||||
lang="en",
|
||||
)
|
||||
|
||||
assert token_ids.tolist() == [[0, 58838, 42, 1]]
|
||||
assert language_id.tolist() == [7]
|
||||
|
||||
|
||||
def test_native_fsdp_checkpoint_keys_are_normalized_at_every_nesting_level():
|
||||
assert _normalize_indextts_checkpoint_key("_fsdp_wrapped_module.spk_emb_proj.weight") == "spk_emb_proj.weight"
|
||||
assert (
|
||||
_normalize_indextts_checkpoint_key("_fsdp_wrapped_module.mel_embedding._fsdp_wrapped_module.weight")
|
||||
== "mel_embedding.weight"
|
||||
)
|
||||
|
||||
|
||||
def test_prefill_span_mismatch_fails_instead_of_padding_or_truncating():
|
||||
for actual_span_len in (9, 11):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="model_type=indextts2_5.*lang=es.*text_token_count=6",
|
||||
):
|
||||
_require_exact_indextts_prefill_span(
|
||||
actual_span_len=actual_span_len,
|
||||
scheduled_span_len=10,
|
||||
model_type="indextts2_5",
|
||||
lang="es",
|
||||
text_token_count=6,
|
||||
)
|
||||
|
||||
|
||||
def test_matching_prefill_span_is_accepted():
|
||||
_require_exact_indextts_prefill_span(
|
||||
actual_span_len=10,
|
||||
scheduled_span_len=10,
|
||||
model_type="indextts2_5",
|
||||
lang="en",
|
||||
text_token_count=6,
|
||||
)
|
||||
|
||||
|
||||
def _make_minimal_talker(
|
||||
monkeypatch,
|
||||
*,
|
||||
model_type,
|
||||
use_gpt_latent,
|
||||
):
|
||||
import vllm_omni.model_executor.models.indextts2.indextts2_talker as talker_module
|
||||
|
||||
seen = {}
|
||||
|
||||
def fake_parallel_lm_head(vocab_size, hidden_size, *, bias=False, **_kwargs):
|
||||
seen["bias"] = bias
|
||||
return torch.nn.Linear(hidden_size, vocab_size, bias=bias)
|
||||
|
||||
monkeypatch.setattr(talker_module, "ParallelLMHead", fake_parallel_lm_head)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"make_layers",
|
||||
lambda *_args, **_kwargs: (
|
||||
0,
|
||||
1,
|
||||
torch.nn.ModuleList([torch.nn.Linear(8, 8)]),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_last_rank=lambda: True),
|
||||
)
|
||||
monkeypatch.setattr(talker_module, "get_speaker_cache", object)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"LogitsProcessor",
|
||||
lambda *_args, **_kwargs: object(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"ConformerEncoder",
|
||||
lambda **_kwargs: torch.nn.Linear(1, 1),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"PerceiverResampler",
|
||||
lambda *_args, **_kwargs: torch.nn.Linear(1, 1),
|
||||
)
|
||||
|
||||
gpt = {
|
||||
"model_dim": 8,
|
||||
"layers": 0,
|
||||
"heads": 1,
|
||||
"max_mel_tokens": 4,
|
||||
"max_text_tokens": 4,
|
||||
"number_mel_codes": 16,
|
||||
"start_mel_token": 14,
|
||||
"stop_mel_token": 15,
|
||||
"number_text_tokens": 32,
|
||||
"condition_num_latent": 1,
|
||||
"condition_module": {},
|
||||
"emo_condition_module": {},
|
||||
}
|
||||
config = SimpleNamespace(
|
||||
gpt=gpt,
|
||||
model_type=model_type,
|
||||
use_gpt_latent=use_gpt_latent,
|
||||
gpt_checkpoint="gpt.pth",
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="/model", hf_config=config),
|
||||
cache_config=None,
|
||||
quant_config=None,
|
||||
compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
|
||||
)
|
||||
|
||||
return (
|
||||
IndexTTS2TalkerForConditionalGeneration(vllm_config=vllm_config),
|
||||
seen,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "use_gpt_latent"),
|
||||
[
|
||||
("indextts2", True),
|
||||
("indextts2_5", False),
|
||||
],
|
||||
)
|
||||
def test_mel_head_preserves_official_checkpoint_bias_and_hidden_payload_policy(
|
||||
monkeypatch,
|
||||
model_type,
|
||||
use_gpt_latent,
|
||||
):
|
||||
talker, seen = _make_minimal_talker(
|
||||
monkeypatch,
|
||||
model_type=model_type,
|
||||
use_gpt_latent=use_gpt_latent,
|
||||
)
|
||||
|
||||
assert seen["bias"] is True
|
||||
assert talker.mel_head.bias is not None
|
||||
assert talker.omni_pooler_payload_include_hidden is use_gpt_latent
|
||||
assert talker.omni_payload_at_request_end is (model_type == "indextts2_5")
|
||||
|
||||
|
||||
def test_v2_load_weights_loads_exact_mel_head_bias_parameter(monkeypatch):
|
||||
import vllm_omni.model_executor.models.indextts2.indextts2_talker as talker_module
|
||||
|
||||
talker, _ = _make_minimal_talker(
|
||||
monkeypatch,
|
||||
model_type="indextts2",
|
||||
use_gpt_latent=True,
|
||||
)
|
||||
checkpoint = {
|
||||
name: parameter.detach().clone() for name, parameter in talker.named_parameters(remove_duplicate=False)
|
||||
}
|
||||
expected_bias = torch.arange(
|
||||
talker.number_mel_codes,
|
||||
dtype=talker.mel_head.bias.dtype,
|
||||
)
|
||||
checkpoint["mel_head.bias"] = expected_bias
|
||||
with torch.no_grad():
|
||||
talker.mel_head.bias.zero_()
|
||||
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"resolve_model_file",
|
||||
lambda *_args, **_kwargs: "/model/gpt.pth",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker_module.torch,
|
||||
"load",
|
||||
lambda *_args, **_kwargs: checkpoint,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker_module,
|
||||
"is_pp_missing_parameter",
|
||||
lambda *_args, **_kwargs: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker,
|
||||
"_warmup_external_models",
|
||||
lambda _device: None,
|
||||
)
|
||||
|
||||
loaded = talker.load_weights([])
|
||||
|
||||
assert "mel_head.bias" in loaded
|
||||
torch.testing.assert_close(talker.mel_head.bias, expected_bias)
|
||||
|
||||
|
||||
def test_compute_logits_passes_mel_head_bias_to_logits_processor():
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
torch.nn.Module.__init__(talker)
|
||||
talker.mel_head = torch.nn.Linear(3, 4, bias=True)
|
||||
hidden_states = torch.zeros(1, 3)
|
||||
seen = {}
|
||||
|
||||
def fake_logits_processor(head, hidden, **kwargs):
|
||||
seen["head"] = head
|
||||
seen["hidden"] = hidden
|
||||
seen.update(kwargs)
|
||||
return hidden
|
||||
|
||||
talker.logits_processor = fake_logits_processor
|
||||
|
||||
result = talker.compute_logits(hidden_states)
|
||||
|
||||
assert result is hidden_states
|
||||
assert seen["head"] is talker.mel_head
|
||||
assert seen["hidden"] is hidden_states
|
||||
assert seen["embedding_bias"] is talker.mel_head.bias
|
||||
|
||||
|
||||
def _make_cached_v25_prefill_talker(monkeypatch):
|
||||
talker, _ = _make_minimal_talker(
|
||||
monkeypatch,
|
||||
model_type="indextts2_5",
|
||||
use_gpt_latent=False,
|
||||
)
|
||||
talker._speaker_cache = SimpleNamespace(
|
||||
make_cache_key=lambda *_args: "cached-speaker",
|
||||
get=lambda _key: {
|
||||
"S_ref": torch.ones(1, 3, 8),
|
||||
"ref_mel": torch.ones(1, 80, 3),
|
||||
"style": torch.ones(1, 192),
|
||||
"spk_cond_emb": torch.ones(1, 3, 8),
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker,
|
||||
"_compute_emotion_vector",
|
||||
lambda **_kwargs: torch.zeros(1, talker.model_dim),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
talker,
|
||||
"_tokenize_text",
|
||||
lambda *_args, **_kwargs: (torch.tensor([[1]]), None),
|
||||
)
|
||||
return talker
|
||||
|
||||
|
||||
def test_prefill_preserves_duration_factor_in_metadata(monkeypatch):
|
||||
talker = _make_cached_v25_prefill_talker(monkeypatch)
|
||||
|
||||
_, _, info_update = talker._preprocess_prefill(
|
||||
torch.zeros(5, dtype=torch.long),
|
||||
None,
|
||||
{
|
||||
"text": ["hello"],
|
||||
"voice": ["/speaker.wav"],
|
||||
"voice_name": ["cached"],
|
||||
"duration_factor": [0.5],
|
||||
},
|
||||
span_len=5,
|
||||
)
|
||||
|
||||
assert info_update["meta"]["duration_factor"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration_factor", [0.49, 2.01])
|
||||
def test_prefill_rejects_duration_factor_outside_supported_range(monkeypatch, duration_factor):
|
||||
talker = _make_cached_v25_prefill_talker(monkeypatch)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="^IndexTTS 2.5 duration_factor must be between 0.5 and 2.0$",
|
||||
):
|
||||
talker._preprocess_prefill(
|
||||
torch.zeros(5, dtype=torch.long),
|
||||
None,
|
||||
{
|
||||
"text": ["hello"],
|
||||
"voice": ["/speaker.wav"],
|
||||
"voice_name": ["cached"],
|
||||
"duration_factor": [duration_factor],
|
||||
},
|
||||
span_len=5,
|
||||
)
|
||||
|
||||
|
||||
def test_no_latent_sampled_token_payload_emits_only_first_step_metadata():
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
talker.use_gpt_latent = False
|
||||
hidden = torch.zeros(1, 8)
|
||||
info = {
|
||||
"codes": {"mel": torch.tensor([17])},
|
||||
"meta": {
|
||||
"mel_code_count": 1,
|
||||
"S_ref": torch.ones(1, 4),
|
||||
"ref_mel": torch.ones(1, 80, 3),
|
||||
"style": torch.ones(1, 192),
|
||||
"duration_factor": 0.5,
|
||||
},
|
||||
}
|
||||
|
||||
first_decode_output = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=[info],
|
||||
)
|
||||
|
||||
assert "codes" not in first_decode_output.multimodal_outputs
|
||||
assert "hidden_states" not in first_decode_output.multimodal_outputs
|
||||
assert first_decode_output.multimodal_outputs["meta"][0]["S_ref"] is info["meta"]["S_ref"]
|
||||
assert first_decode_output.multimodal_outputs["meta"][0]["duration_factor"] == 0.5
|
||||
|
||||
later = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=[
|
||||
{
|
||||
"codes": {"mel": torch.tensor([23])},
|
||||
"meta": {"mel_code_count": 2},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert later.multimodal_outputs == {"meta": [{"use_gpt_latent": False}]}
|
||||
|
||||
|
||||
def test_first_decode_payload_defaults_missing_duration_factor_to_one():
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
talker.use_gpt_latent = False
|
||||
hidden = torch.zeros(1, 8)
|
||||
info = {
|
||||
"codes": {"mel": torch.tensor([17])},
|
||||
"meta": {
|
||||
"mel_code_count": 1,
|
||||
"S_ref": torch.ones(1, 4),
|
||||
"ref_mel": torch.ones(1, 80, 3),
|
||||
"style": torch.ones(1, 192),
|
||||
},
|
||||
}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=[info],
|
||||
)
|
||||
|
||||
assert output.multimodal_outputs["meta"][0]["duration_factor"] == 1.0
|
||||
|
||||
|
||||
def test_latent_sampled_token_payload_keeps_aligned_gpu_deltas():
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
talker.use_gpt_latent = True
|
||||
hidden = torch.zeros(1, 8)
|
||||
info = {
|
||||
"codes": {"mel": torch.tensor([17])},
|
||||
"meta": {
|
||||
"mel_code_count": 1,
|
||||
"latent_acc": torch.arange(8, dtype=torch.float32).reshape(1, 8),
|
||||
"duration_factor": 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
output = talker.make_omni_output(
|
||||
hidden,
|
||||
model_intermediate_buffer=[info],
|
||||
)
|
||||
|
||||
assert output.multimodal_outputs["codes"]["mel"][0].tolist() == [[17]]
|
||||
assert output.multimodal_outputs["hidden_states"]["latent"][0].tolist() == [list(range(8))]
|
||||
|
||||
|
||||
def test_v25_default_emotion_reuses_speaker_w2v_features(monkeypatch):
|
||||
talker = object.__new__(IndexTTS2TalkerForConditionalGeneration)
|
||||
speaker_features = torch.zeros(1, 4, 1024)
|
||||
expected = torch.ones(1, 1280)
|
||||
seen = {}
|
||||
|
||||
def fake_merge(spk_cond_emb, emo_cond_emb, alpha):
|
||||
seen["spk"] = spk_cond_emb
|
||||
seen["emo"] = emo_cond_emb
|
||||
seen["alpha"] = alpha
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(talker, "_merge_emovec", fake_merge)
|
||||
|
||||
result = talker._compute_emotion_vector(
|
||||
wav_16k=None,
|
||||
speaker_audio_path="/speaker.wav",
|
||||
emo_audio_path=None,
|
||||
emo_voice_name=None,
|
||||
main_text="hello",
|
||||
use_emo_text=False,
|
||||
emo_text=None,
|
||||
emo_vector=None,
|
||||
emo_alpha=0.25,
|
||||
use_random=False,
|
||||
style=torch.zeros(1, 192),
|
||||
spk_cond_emb=speaker_features,
|
||||
w2v_model=None,
|
||||
w2v_proc=None,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert result is expected
|
||||
assert seen == {
|
||||
"spk": speaker_features,
|
||||
"emo": speaker_features,
|
||||
"alpha": 1.0,
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import builtins
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2 import text_processing_v2_5
|
||||
from vllm_omni.model_executor.models.indextts2.text_processing_v2_5 import (
|
||||
apply_pronunciation_annotations,
|
||||
clean_indextts25_text,
|
||||
prepare_indextts25_text,
|
||||
)
|
||||
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
|
||||
INDEXTTS25_SPECIAL_TOKENS,
|
||||
INDEXTTS25_VOCAB_SIZE,
|
||||
LANGUAGE_DICT,
|
||||
lang_to_token,
|
||||
normalize_language_code,
|
||||
resolve_indextts25_tokenizer_file,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_special_token_layout_matches_official_checkpoint_vocab():
|
||||
assert len(INDEXTTS25_SPECIAL_TOKENS) == 1673
|
||||
assert len(set(INDEXTTS25_SPECIAL_TOKENS)) == 1673
|
||||
assert 58836 + len(INDEXTTS25_SPECIAL_TOKENS) == INDEXTTS25_VOCAB_SIZE
|
||||
assert INDEXTTS25_VOCAB_SIZE == 60509
|
||||
|
||||
|
||||
def test_pronunciation_annotations_select_english_and_chinese_markers():
|
||||
assert apply_pronunciation_annotations("<going|G OW1 . IH0 NG>") == (
|
||||
"<|SPECIAL_TOKEN_1|>G OW1 . IH0 NG<|SPECIAL_TOKEN_1|>"
|
||||
)
|
||||
assert apply_pronunciation_annotations("<行|xing2>") == ("<|SPECIAL_TOKEN_2|>XING2<|SPECIAL_TOKEN_2|>")
|
||||
|
||||
|
||||
def test_prepare_text_applies_language_prefix_case_and_annotation(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_encode(text, *, model_dir, tokenizer_file):
|
||||
captured["text"] = text
|
||||
captured["model_dir"] = model_dir
|
||||
captured["tokenizer_file"] = tokenizer_file
|
||||
return [7, 8, 9]
|
||||
|
||||
monkeypatch.setattr(text_processing_v2_5, "encode_indextts25_text", fake_encode)
|
||||
|
||||
token_ids, lang_id = prepare_indextts25_text(
|
||||
"HELLO <行|xing2>",
|
||||
lang="zh",
|
||||
model_dir="/model",
|
||||
text_normalization=False,
|
||||
)
|
||||
|
||||
assert token_ids == [7, 8, 9]
|
||||
assert lang_id == LANGUAGE_DICT["zh"]
|
||||
assert captured == {
|
||||
"text": "<|zh|> hello <|SPECIAL_TOKEN_2|>XING2<|SPECIAL_TOKEN_2|>",
|
||||
"model_dir": "/model",
|
||||
"tokenizer_file": "multilingual_zh_ja_yue_char_del.tiktoken",
|
||||
}
|
||||
|
||||
|
||||
def test_mandarin_alias_is_an_intentional_vllm_omni_convenience():
|
||||
assert normalize_language_code("Mandarin") == "zh"
|
||||
assert lang_to_token("mandarin") == LANGUAGE_DICT["zh"]
|
||||
|
||||
|
||||
def test_invalid_language_does_not_fall_back_to_common():
|
||||
with pytest.raises(ValueError, match="Unsupported IndexTTS 2.5 language"):
|
||||
lang_to_token("xx-invalid")
|
||||
|
||||
|
||||
def test_zhen_uses_mixed_normalization():
|
||||
calls = []
|
||||
|
||||
def fake_mixed_normalizer(text):
|
||||
calls.append(text)
|
||||
return "MIXED"
|
||||
|
||||
original = text_processing_v2_5._normalize_zh_or_en
|
||||
text_processing_v2_5._normalize_zh_or_en = fake_mixed_normalizer
|
||||
try:
|
||||
result = text_processing_v2_5._normalize_with_official_backend(
|
||||
"中文 and English",
|
||||
"zhen",
|
||||
)
|
||||
finally:
|
||||
text_processing_v2_5._normalize_zh_or_en = original
|
||||
|
||||
assert calls == ["中文 and English"]
|
||||
assert result == "MIXED"
|
||||
|
||||
|
||||
def test_spanish_normalization_does_not_hide_broken_dependencies(monkeypatch):
|
||||
original_import = builtins.__import__
|
||||
|
||||
def controlled_import(name, *args, **kwargs):
|
||||
if name == "nemo_text_processing.text_normalization.normalize":
|
||||
raise RuntimeError("broken NeMo grammar")
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", controlled_import)
|
||||
|
||||
with pytest.raises(RuntimeError, match="broken NeMo grammar"):
|
||||
text_processing_v2_5._normalize_with_official_backend("hola", "es")
|
||||
|
||||
|
||||
def test_zhen_keeps_literal_prefix_and_uses_common_embedding(monkeypatch):
|
||||
captured = {}
|
||||
|
||||
def fake_encode(text, *, model_dir, tokenizer_file):
|
||||
captured["text"] = text
|
||||
return [7, 8, 9]
|
||||
|
||||
monkeypatch.setattr(text_processing_v2_5, "encode_indextts25_text", fake_encode)
|
||||
|
||||
token_ids, lang_id = prepare_indextts25_text(
|
||||
"HELLO 中文",
|
||||
lang="zhen",
|
||||
model_dir="/model",
|
||||
text_normalization=False,
|
||||
)
|
||||
|
||||
assert token_ids == [7, 8, 9]
|
||||
assert captured["text"] == "<|zhen|> hello 中文"
|
||||
assert lang_id == LANGUAGE_DICT["common"]
|
||||
assert "<|zhen|>" not in INDEXTTS25_SPECIAL_TOKENS
|
||||
assert "<|common|>" not in INDEXTTS25_SPECIAL_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
(",,,", "…"),
|
||||
("测试,,,结束", "测试…结束"),
|
||||
],
|
||||
)
|
||||
def test_repeated_full_width_commas_use_longest_first_cleanup(text, expected):
|
||||
assert clean_indextts25_text(text) == expected
|
||||
|
||||
|
||||
def test_default_text_normalizer_is_shared_by_prompt_and_talker(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_normalizer(text, lang):
|
||||
calls.append((text, lang))
|
||||
return "twenty five percent"
|
||||
|
||||
monkeypatch.setattr(
|
||||
text_processing_v2_5,
|
||||
"_normalize_with_official_backend",
|
||||
fake_normalizer,
|
||||
)
|
||||
|
||||
result = text_processing_v2_5.normalize_indextts25_text(
|
||||
"25%",
|
||||
lang="en",
|
||||
text_normalization=True,
|
||||
)
|
||||
|
||||
assert calls == [("25%", "en")]
|
||||
assert result == "twenty five percent"
|
||||
|
||||
|
||||
def test_mixed_text_uses_official_content_routing_and_protection(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class FakeNormalizer:
|
||||
def __init__(self, lang):
|
||||
self.lang = lang
|
||||
|
||||
def normalize(self, text):
|
||||
calls.append((self.lang, text))
|
||||
return text.replace("<H>", " <H> ").replace("5", "five")
|
||||
|
||||
monkeypatch.setattr(
|
||||
text_processing_v2_5,
|
||||
"_load_wetext_normalizer",
|
||||
lambda lang: FakeNormalizer(lang),
|
||||
)
|
||||
|
||||
result = text_processing_v2_5._normalize_with_official_backend(
|
||||
"GPT-5-nano 中文 xuan4",
|
||||
"en",
|
||||
)
|
||||
|
||||
assert calls[0][0] == "zh"
|
||||
assert result == "GPT-five-nano 中文 XVAN4"
|
||||
|
||||
|
||||
def test_tokenizer_resolves_native_checkpoints_subdirectory(tmp_path):
|
||||
tokenizer = tmp_path / "checkpoints" / "multilingual_zh_ja_yue_char_del.tiktoken"
|
||||
tokenizer.parent.mkdir()
|
||||
tokenizer.touch()
|
||||
|
||||
assert resolve_indextts25_tokenizer_file(str(tmp_path)) == str(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("lang", "text", "expected"),
|
||||
[
|
||||
(
|
||||
"es",
|
||||
"En la actualidad se continúa utilizando una corona abierta.",
|
||||
"<|es|> EN LA ACTUALIDAD SE CONTINÚA UTILIZANDO UNA CORONA ABIERTA.",
|
||||
),
|
||||
(
|
||||
"ar",
|
||||
"فكلُّها خياراتٌ اتَّخذتَها بنفسِك.",
|
||||
"<|ar|> فكلُّها خياراتٌ اتَّخذتَها بنفسِك.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_official_spanish_and_arabic_cases_preserve_expected_text_contract(
|
||||
monkeypatch,
|
||||
lang,
|
||||
text,
|
||||
expected,
|
||||
):
|
||||
captured = {}
|
||||
|
||||
def fake_encode(value, *, model_dir, tokenizer_file):
|
||||
captured["text"] = value
|
||||
return [1]
|
||||
|
||||
monkeypatch.setattr(
|
||||
text_processing_v2_5,
|
||||
"encode_indextts25_text",
|
||||
fake_encode,
|
||||
)
|
||||
|
||||
_, lang_id = prepare_indextts25_text(
|
||||
text,
|
||||
lang=lang,
|
||||
model_dir="/model",
|
||||
text_normalization=False,
|
||||
)
|
||||
|
||||
assert captured["text"] == expected
|
||||
assert lang_id == LANGUAGE_DICT[lang]
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.indextts2.indextts2_s2mel_decoder import (
|
||||
IndexTTS2S2MelDecoder,
|
||||
)
|
||||
from vllm_omni.model_executor.models.indextts2.s2mel.modules.wavenet import WN
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_wavenet() -> WN:
|
||||
torch.manual_seed(0)
|
||||
return WN(
|
||||
hidden_channels=2,
|
||||
kernel_size=5,
|
||||
dilation_rate=1,
|
||||
n_layers=8,
|
||||
gin_channels=2,
|
||||
p_dropout=0.0,
|
||||
causal=False,
|
||||
).eval()
|
||||
|
||||
|
||||
def _sequence_mask(lengths: torch.Tensor, max_length: int) -> torch.Tensor:
|
||||
positions = torch.arange(max_length, device=lengths.device)
|
||||
return (positions.unsqueeze(0) < lengths.unsqueeze(1)).unsqueeze(1)
|
||||
|
||||
|
||||
def _singleton_outputs(
|
||||
wavenet: WN,
|
||||
inputs: torch.Tensor,
|
||||
lengths: list[int],
|
||||
conditioning: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
# This is the independent oracle: every request ends at the physical tensor
|
||||
# boundary and therefore uses SConv1d's original reflect-padding behavior.
|
||||
wavenet._assume_full_mask = True
|
||||
return [
|
||||
wavenet(
|
||||
inputs[row : row + 1, :, :length],
|
||||
torch.ones(1, 1, length, dtype=torch.bool),
|
||||
g=conditioning[row : row + 1],
|
||||
)
|
||||
for row, length in enumerate(lengths)
|
||||
]
|
||||
|
||||
|
||||
def test_ragged_wavenet_matches_original_singleton_reflect_boundaries():
|
||||
wavenet = _make_wavenet()
|
||||
lengths = [6, 5, 4, 3]
|
||||
inputs = torch.randn(len(lengths), 2, max(lengths))
|
||||
conditioning = torch.randn(len(lengths), 2, 1)
|
||||
expected = _singleton_outputs(wavenet, inputs, lengths, conditioning)
|
||||
|
||||
mask = _sequence_mask(torch.tensor(lengths), inputs.shape[-1])
|
||||
wavenet._assume_full_mask = False
|
||||
# Keep deliberately non-zero invalid tails. The first WaveNet layer sees
|
||||
# transformer output before the recurrent mask has cleared those values.
|
||||
actual = wavenet(inputs, mask, g=conditioning)
|
||||
|
||||
for row, (length, singleton) in enumerate(zip(lengths, expected)):
|
||||
torch.testing.assert_close(
|
||||
actual[row : row + 1, :, :length],
|
||||
singleton,
|
||||
rtol=1e-5,
|
||||
atol=1e-6,
|
||||
)
|
||||
|
||||
|
||||
def test_wavenet_exposes_minimum_safe_ragged_length():
|
||||
# kernel=5, dilation=1 has two right-reflect ghost positions, so a
|
||||
# logical row must contain at least three frames before it can use the
|
||||
# vectorized ragged path. Shorter rows stay on original singleton SConv1d.
|
||||
assert _make_wavenet().ragged_min_length == 3
|
||||
|
||||
|
||||
def test_cfm_grouping_isolates_rows_too_short_for_logical_reflect():
|
||||
groups = IndexTTS2S2MelDecoder._cfm_group_indices(
|
||||
sorted_indices=[0, 1, 2, 3, 4],
|
||||
total_lengths=[0, 1, 2, 3, 6],
|
||||
max_group_size=2,
|
||||
min_ragged_length=3,
|
||||
)
|
||||
|
||||
assert groups == [[0], [1], [2], [3, 4]]
|
||||
|
||||
|
||||
def test_ragged_wavenet_preserves_cfg_cond_uncond_row_order():
|
||||
wavenet = _make_wavenet()
|
||||
base_lengths = [3, 6]
|
||||
conditional = torch.randn(2, 2, 6)
|
||||
unconditional = torch.randn(2, 2, 6)
|
||||
cfg_inputs = torch.cat([conditional, unconditional], dim=0)
|
||||
cfg_lengths = base_lengths + base_lengths
|
||||
cfg_conditioning = torch.randn(4, 2, 1)
|
||||
expected = _singleton_outputs(
|
||||
wavenet,
|
||||
cfg_inputs,
|
||||
cfg_lengths,
|
||||
cfg_conditioning,
|
||||
)
|
||||
|
||||
mask = _sequence_mask(torch.tensor(cfg_lengths), cfg_inputs.shape[-1])
|
||||
wavenet._assume_full_mask = False
|
||||
actual = wavenet(cfg_inputs, mask, g=cfg_conditioning)
|
||||
|
||||
for row, (length, singleton) in enumerate(zip(cfg_lengths, expected)):
|
||||
torch.testing.assert_close(
|
||||
actual[row : row + 1, :, :length],
|
||||
singleton,
|
||||
rtol=1e-5,
|
||||
atol=1e-6,
|
||||
)
|
||||
|
||||
|
||||
def test_full_mask_wavenet_path_is_bit_exact_with_original_path():
|
||||
wavenet = _make_wavenet()
|
||||
inputs = torch.randn(3, 2, 6)
|
||||
mask = torch.ones(3, 1, 6, dtype=torch.bool)
|
||||
conditioning = torch.randn(3, 2, 1)
|
||||
|
||||
if hasattr(wavenet, "_assume_full_mask"):
|
||||
del wavenet._assume_full_mask
|
||||
expected = wavenet(inputs, mask, g=conditioning)
|
||||
|
||||
wavenet._assume_full_mask = True
|
||||
actual = wavenet(inputs, mask, g=conditioning)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(torch, "compile"), reason="torch.compile unavailable")
|
||||
def test_ragged_wavenet_is_one_dynamic_fullgraph():
|
||||
wavenet = _make_wavenet()
|
||||
lengths = [3, 6]
|
||||
inputs = torch.randn(2, 2, 6)
|
||||
conditioning = torch.randn(2, 2, 1)
|
||||
mask = _sequence_mask(torch.tensor(lengths), inputs.shape[-1])
|
||||
expected = _singleton_outputs(wavenet, inputs, lengths, conditioning)
|
||||
wavenet._assume_full_mask = False
|
||||
|
||||
compiled = torch.compile(
|
||||
wavenet,
|
||||
backend="eager",
|
||||
fullgraph=True,
|
||||
dynamic=True,
|
||||
)
|
||||
actual = compiled(inputs, mask, g=conditioning)
|
||||
|
||||
for row, (length, singleton) in enumerate(zip(lengths, expected)):
|
||||
torch.testing.assert_close(
|
||||
actual[row : row + 1, :, :length],
|
||||
singleton,
|
||||
rtol=1e-5,
|
||||
atol=1e-6,
|
||||
)
|
||||
|
||||
|
||||
def test_decoder_full_mask_switch_also_controls_wavenet_boundary_path():
|
||||
decoder = object.__new__(IndexTTS2S2MelDecoder)
|
||||
attention_layers = [SimpleNamespace(attention=SimpleNamespace()) for _ in range(2)]
|
||||
estimator = SimpleNamespace(
|
||||
transformer=SimpleNamespace(layers=attention_layers),
|
||||
wavenet=SimpleNamespace(),
|
||||
)
|
||||
|
||||
decoder._set_dit_full_mask_fast_path(estimator, enabled=False)
|
||||
|
||||
assert all(layer.attention._assume_full_mask is False for layer in attention_layers)
|
||||
assert estimator.wavenet._assume_full_mask is False
|
||||
|
||||
decoder._set_dit_full_mask_fast_path(estimator, enabled=True)
|
||||
|
||||
assert all(layer.attention._assume_full_mask is True for layer in attention_layers)
|
||||
assert estimator.wavenet._assume_full_mask is True
|
||||
|
||||
|
||||
def test_decoder_full_mask_switch_unwraps_compiled_estimator():
|
||||
decoder = object.__new__(IndexTTS2S2MelDecoder)
|
||||
attention_layers = [SimpleNamespace(attention=SimpleNamespace()) for _ in range(2)]
|
||||
eager_estimator = SimpleNamespace(
|
||||
transformer=SimpleNamespace(layers=attention_layers),
|
||||
wavenet=SimpleNamespace(),
|
||||
)
|
||||
compiled_estimator = SimpleNamespace(_orig_mod=eager_estimator)
|
||||
|
||||
decoder._set_dit_full_mask_fast_path(compiled_estimator, enabled=False)
|
||||
|
||||
assert all(layer.attention._assume_full_mask is False for layer in attention_layers)
|
||||
assert eager_estimator.wavenet._assume_full_mask is False
|
||||
Reference in new issue
Block a user