[TTS][Model] support MOSS-TTS series (#3420)

Signed-off-by: Zhang <jianmusings@gmail.com>
Signed-off-by: Zhang Jian <jianmusings@gmail.com>
Signed-off-by: jian <jianmusings@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Co-authored-by: Canlin Guo <961750412@qq.com>
This commit is contained in:
Zhang JianandCanlin Guo authored and GitHub committed 2026-06-02 15:46:01 +08:00
1 parent da9c45e1de
commit b550709bba
32 files changed
+4775 -37

No files matched your search

@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E offline inference tests for MOSS-TTS two-stage pipeline.
Tests the MossTTSDelayModel path using MOSS-VoiceGenerator (1.7B) — the
smallest MossTTSDelayModel variant — so the test runs on a single A10G or L4
without requiring an 80 GB GPU.
A second test class covers MOSS-TTS-Realtime (1.7B, MossTTSRealtime) to
exercise the local-transformer generation path.
Both classes use a module-scoped engine so the model is loaded once per test
module execution. Do NOT add a second engine fixture in this file — that
triggers mid-module teardown races (see skill invariant I4).
"""
from __future__ import annotations
import os
import urllib.request
import pytest
import torch
from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from vllm_omni import Omni
# ---------------------------------------------------------------------------
# Shared constants
# ---------------------------------------------------------------------------
# Selected by the nightly "TTS · Function Test" (-m "full_model and L4 and tts").
# Without the tts + run-level marks these tests are collected but never run.
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
SAMPLE_RATE = 24_000 # All MOSS-TTS full variants output 24 kHz
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
_DEFAULT_SAMPLING = SamplingParams(
temperature=1.7,
top_p=0.8,
top_k=25,
max_tokens=512, # short for tests (~20 s at 24 kHz, ~12.5 tokens/s)
seed=42,
detokenize=False,
)
# ---------------------------------------------------------------------------
# Session fixtures
# ---------------------------------------------------------------------------
@pytest.fixture(scope="session")
def ref_audio_path(tmp_path_factory) -> str:
"""Download the upstream reference clip once per session.
Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to skip in air-gapped environments.
"""
cache_dir = tmp_path_factory.mktemp("moss_tts_ref")
target = cache_dir / "zh_1.wav"
try:
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
target.write_bytes(resp.read())
except Exception as exc:
msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}"
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
pytest.skip(msg)
pytest.fail(msg)
if not target.exists() or target.stat().st_size == 0:
pytest.fail(f"Reference clip empty after download: {target}")
return str(target)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _build_request(text: str, ref_audio_path: str, seed: int = 42) -> dict:
return {
"prompt": "<|im_start|>assistant\n",
"additional_information": {
"task_type": ["voice_clone"],
"text": [text],
"mode": ["voice_clone"],
"prompt_audio_path": [ref_audio_path],
"seed": [seed],
},
}
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
"""Run one request; return (waveform_cpu, sample_rate)."""
for stage_outputs in omni.generate(request, _DEFAULT_SAMPLING):
for req_output in stage_outputs.request_output:
mm = req_output.outputs[0].multimodal_output
assert mm is not None, "Expected non-None multimodal_output"
# Never use ``a or b`` on tensor values — a multi-element tensor
# raises on truthiness. Check ``is None`` explicitly and handle the
# pre-consolidation list form.
audio = mm.get("audio")
if audio is None:
audio = mm.get("model_outputs")
sr = mm.get("sr")
assert audio is not None, "Expected 'audio' or 'model_outputs' in multimodal_output"
if isinstance(audio, list):
audio = torch.cat(
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
dim=0,
)
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
raise AssertionError("No stage outputs received")
# ---------------------------------------------------------------------------
# MossTTSDelayModel — MOSS-VoiceGenerator 1.7B
# (tests the delay-pattern generation path on a small GPU)
# ---------------------------------------------------------------------------
_DELAY_MODEL = "OpenMOSS-Team/MOSS-VoiceGenerator"
@pytest.fixture(scope="module")
def delay_engine():
return Omni(model=_DELAY_MODEL, stage_init_timeout=300)
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_delay_english(delay_engine, ref_audio_path):
"""MossTTSDelayModel: English voice_clone produces non-empty 24 kHz audio."""
req = _build_request("Hello, this is a MOSS-TTS voice cloning test.", ref_audio_path)
audio, sr = _collect_audio(delay_engine, req)
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
assert audio.numel() > 0, "Audio tensor is empty"
assert not torch.all(audio == 0), "Audio is silence"
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_delay_chinese(delay_engine, ref_audio_path):
"""MossTTSDelayModel: Chinese input produces non-empty audio."""
req = _build_request("你好,这是语音合成测试。", ref_audio_path)
audio, sr = _collect_audio(delay_engine, req)
assert sr == SAMPLE_RATE
assert audio.numel() > 0
assert not torch.all(audio == 0)
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_delay_deterministic(delay_engine, ref_audio_path):
"""MossTTSDelayModel: same seed yields identical waveforms."""
req = _build_request("Reproducibility check.", ref_audio_path, seed=99)
audio1, _ = _collect_audio(delay_engine, req)
audio2, _ = _collect_audio(delay_engine, req)
assert audio1.shape == audio2.shape, "Shapes differ across identical seeds"
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms differ across identical seeds"
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_delay_batch(delay_engine, ref_audio_path):
"""MossTTSDelayModel: batch of two requests each returns non-empty audio."""
requests = [
_build_request("First sentence.", ref_audio_path),
_build_request("Second sentence.", ref_audio_path),
]
results: list[torch.Tensor] = []
for stage_outputs in delay_engine.generate(requests, [_DEFAULT_SAMPLING] * 2):
for req_output in stage_outputs.request_output:
mm = req_output.outputs[0].multimodal_output
assert mm is not None
audio = mm.get("audio")
if audio is None:
audio = mm.get("model_outputs")
assert audio is not None
if isinstance(audio, list):
audio = torch.cat(
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
dim=0,
)
results.append(audio.reshape(-1).cpu())
assert len(results) == 2, f"Expected 2 outputs, got {len(results)}"
for i, audio in enumerate(results):
assert audio.numel() > 0, f"Audio[{i}] is empty"
# ---------------------------------------------------------------------------
# MossTTSRealtime — MOSS-TTS-Realtime 1.7B
# (tests the local-transformer generation path)
# ---------------------------------------------------------------------------
_REALTIME_MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
@pytest.fixture(scope="module")
def realtime_engine():
return Omni(model=_REALTIME_MODEL, stage_init_timeout=300)
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_realtime_english(realtime_engine, ref_audio_path):
"""MossTTSRealtime: English voice_clone produces non-empty 24 kHz audio."""
req = _build_request("This is a real-time TTS streaming test.", ref_audio_path)
audio, sr = _collect_audio(realtime_engine, req)
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
assert audio.numel() > 0, "Audio tensor is empty"
assert not torch.all(audio == 0), "Audio is silence"
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_realtime_deterministic(realtime_engine, ref_audio_path):
"""MossTTSRealtime: same seed yields identical waveforms."""
req = _build_request("Determinism check for realtime model.", ref_audio_path, seed=7)
audio1, _ = _collect_audio(realtime_engine, req)
audio2, _ = _collect_audio(realtime_engine, req)
assert audio1.shape == audio2.shape
assert torch.allclose(audio1, audio2, atol=1e-4)
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E offline inference test for MOSS-TTS-v1.5 (MossTTSDelay-8B).
MOSS-TTS-v1.5 is a continued-training upgrade of MOSS-TTS 1.0 with the SAME
``MossTTSDelay`` architecture and API, so it reuses the existing talker/codec
path unchanged — this test just pins that the 8B v1.5 checkpoint loads and
produces audio.
The model is 8B, so this lives in its own file (one module-scoped engine per
file, per skill invariant I4) and is gated behind an 80 GB GPU; the small-GPU
MossTTSDelay coverage stays in ``test_moss_tts.py`` (1.7B VoiceGenerator).
"""
from __future__ import annotations
import os
import urllib.request
import pytest
import torch
from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from vllm_omni import Omni
# H100-gated (8B). Picked up by the nightly H100 omni job
# (-m "full_model and H100 and omni"); the tts mark keeps it consistent with
# the other MOSS-TTS tests and any tts-scoped sweeps.
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
SAMPLE_RATE = 24_000
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/main/assets/audio/reference_zh_1.wav"
_MODEL = "OpenMOSS-Team/MOSS-TTS-v1.5"
_DEFAULT_SAMPLING = SamplingParams(
temperature=1.7,
top_p=0.8,
top_k=25,
max_tokens=512, # short for tests (~20 s at 24 kHz, ~12.5 tokens/s)
seed=42,
detokenize=False,
)
@pytest.fixture(scope="session")
def ref_audio_path(tmp_path_factory) -> str:
"""Download the upstream reference clip once per session.
Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to skip in air-gapped environments.
"""
target = tmp_path_factory.mktemp("moss_tts_v15_ref") / "zh_1.wav"
try:
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
target.write_bytes(resp.read())
except Exception as exc: # noqa: BLE001
msg = f"Cannot fetch reference clip {REF_AUDIO_URL}: {exc}"
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
pytest.skip(msg)
pytest.fail(msg)
if not target.exists() or target.stat().st_size == 0:
pytest.fail(f"Reference clip empty after download: {target}")
return str(target)
@pytest.fixture(scope="module")
def v15_engine():
return Omni(model=_MODEL, stage_init_timeout=600)
def _build_request(text: str, ref_audio_path: str) -> dict:
return {
"prompt": "<|im_start|>assistant\n",
"additional_information": {
"task_type": ["voice_clone"],
"text": [text],
"mode": ["voice_clone"],
"prompt_audio_path": [ref_audio_path],
"seed": [42],
},
}
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
for stage_outputs in omni.generate(request, _DEFAULT_SAMPLING):
for req_output in stage_outputs.request_output:
mm = req_output.outputs[0].multimodal_output
assert mm is not None, "Expected non-None multimodal_output"
audio = mm.get("audio")
if audio is None:
audio = mm.get("model_outputs")
sr = mm.get("sr")
assert audio is not None, "Expected 'audio' or 'model_outputs' in multimodal_output"
if isinstance(audio, list):
audio = torch.cat(
[t.reshape(-1) for t in audio if isinstance(t, torch.Tensor) and t.numel() > 0],
dim=0,
)
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
return audio.reshape(-1).cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
raise AssertionError("No stage outputs received")
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"})
def test_moss_tts_v15_voice_clone(v15_engine, ref_audio_path):
"""MOSS-TTS-v1.5 (8B): voice_clone produces non-empty 24 kHz audio."""
req = _build_request("Hello, this is a MOSS-TTS v1.5 voice cloning test.", ref_audio_path)
audio, sr = _collect_audio(v15_engine, req)
assert sr == SAMPLE_RATE, f"Expected {SAMPLE_RATE} Hz, got {sr}"
assert audio.numel() > 0, "Audio tensor is empty"
assert not torch.all(audio == 0), "Audio is silence"
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online tests for the full MOSS-TTS family via /v1/audio/speech.
The offline suite (``tests/e2e/offline_inference/test_moss_tts.py`` and
``test_moss_tts_v1_5.py``) covers the engine path; this file covers the
*serving* path (``serving_speech.py`` → ``_detect_moss_variant`` → the
delay/realtime prompt builders), which the Nano online test does not exercise.
We use MOSS-TTS-Realtime (1.7B, ``MossTTSRealtime``) because it is the
smallest full-family variant and fits a single L4 — the 8B delay variants
(MOSS-TTS / v1.5) are H100-gated and stay in the offline suite. Realtime uses
the local-transformer generation path and, like the ``tts`` delay variant,
requires ``ref_audio`` for voice cloning.
The reference clip is fetched from the upstream repo
(``assets/audio/reference_zh_1.wav``, ~1.5 MB) and passed as a base64 data URL.
"""
import base64
import os
import urllib.request
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
MODEL = "OpenMOSS-Team/MOSS-TTS-Realtime"
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS/HEAD/assets/audio/reference_zh_1.wav"
@pytest.fixture(scope="session")
def ref_audio_data_url() -> str:
"""Fetch the upstream sample clip and return it as a base64 data URL.
The fetch failure is escalated to a hard failure (not pytest.skip) so a
broken network path does not silently mask regressions in
/v1/audio/speech. Set ``MOSS_TTS_SKIP_ON_NET_FAIL=1`` to opt into skipping
in air-gapped environments.
"""
try:
with urllib.request.urlopen(REF_AUDIO_URL, timeout=30) as resp:
data = resp.read()
except Exception as e: # noqa: BLE001
msg = f"Cannot fetch upstream reference clip {REF_AUDIO_URL}: {e}"
if os.environ.get("MOSS_TTS_SKIP_ON_NET_FAIL"):
pytest.skip(msg)
pytest.fail(msg)
if not data:
pytest.fail(f"Reference clip empty: {REF_AUDIO_URL}")
return f"data:audio/wav;base64,{base64.b64encode(data).decode('ascii')}"
def get_prompt(prompt_type: str = "text") -> str:
"""Plain natural-sounding sentences for text-to-audio tests.
Avoid the model's own name in the input — the codec mispronounces it,
which can trip transcript-similarity checks without a real regression.
"""
prompts = {
"text": "Hello, this is a real-time voice cloning demo for testing.",
"chinese": "你好,这是一段实时语音合成测试。",
}
return prompts.get(prompt_type, prompts["text"])
tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_deploy_config_path("moss_tts_realtime.yaml"),
server_args=["--disable-log-stats"],
),
id="moss_tts_realtime",
)
]
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> None:
"""
Realtime voice_clone via /v1/audio/speech, non-streaming.
Deploy Setting: moss_tts_realtime.yaml
Input Modal: text + reference audio (voice clone)
Output Modal: audio (24 kHz, WAV)
Input Setting: stream=False
Datasets: single request
"""
request_config = {
"model": omni_server.model,
"input": get_prompt(),
"stream": False,
"response_format": "wav",
"ref_audio": ref_audio_data_url,
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_002_streaming(omni_server, openai_client, ref_audio_data_url) -> None:
"""
Realtime voice_clone via /v1/audio/speech, streaming PCM.
Deploy Setting: moss_tts_realtime.yaml
Input Modal: text + reference audio (voice clone)
Output Modal: audio (24 kHz, PCM stream)
Input Setting: stream=True
Datasets: single request
NOTE: ``min_hnr_db=-5.0`` mirrors the Nano streaming case — MOSS
voice_clone output is intrinsically noisy by the HNR metric even when it
sounds correct; the relaxed threshold still catches catastrophic decoder
failures (HNR far below 0) without flaking on normal output.
"""
request_config = {
"model": omni_server.model,
"input": get_prompt(),
"stream": True,
"response_format": "pcm",
"ref_audio": ref_audio_data_url,
"min_hnr_db": -5.0,
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_003_chinese(omni_server, openai_client, ref_audio_data_url) -> None:
"""
Realtime voice_clone via /v1/audio/speech, Chinese input.
Deploy Setting: moss_tts_realtime.yaml
Input Modal: text (Chinese) + reference audio (voice clone)
Output Modal: audio (24 kHz, WAV)
Input Setting: stream=False
Datasets: single request
"""
request_config = {
"model": omni_server.model,
"input": get_prompt("chinese"),
"stream": False,
"response_format": "wav",
"ref_audio": ref_audio_data_url,
}
openai_client.send_audio_speech_request(request_config)