feat(moss-tts-nano): add single-stage MOSS-TTS-Nano TTS support
Integrates OpenMOSS-Team/MOSS-TTS-Nano (0.1B) as a single GPUGenerationWorker stage. Both the AR LM and MOSS-Audio-Tokenizer-Nano codec run inside MossTTSNanoForGeneration.forward(), removing the need for an inter-stage connector. Key design choices: - Weights loaded in load_weights() not __init__ (avoids pre-CUDA alloc) - trust_remote_code delegates to upstream HF model classes - codec path read from config.audio_tokenizer_pretrained_name_or_path - inference_stream() collects progressive audio chunks for low latency - 48 kHz stereo output; voice clone + continuation modes supported Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
1 parent
bc4a659f03
commit
da6b1b5741
7 files changed
+806
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for MOSS-TTS-Nano in vLLM-Omni single-stage pipeline."""
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
class MossTTSNanoConfig(PretrainedConfig):
|
||||
"""Config for MOSS-TTS-Nano (OpenMOSS-Team/MOSS-TTS-Nano).
|
||||
|
||||
The model is a Global-Local GPT2-based AR LM (0.1B) paired with the
|
||||
MOSS-Audio-Tokenizer-Nano codec. Both components run in a single
|
||||
vLLM-Omni generation stage.
|
||||
|
||||
Relevant fields from config.json:
|
||||
gpt2_config – backbone config (n_layer=12, n_embd=768, n_head=12, ...)
|
||||
n_vq – number of RVQ codebooks (16)
|
||||
audio_vocab_size – per-codebook vocabulary size (1024)
|
||||
audio_tokenizer_pretrained_name_or_path – HF hub path for codec model
|
||||
"""
|
||||
|
||||
model_type = "moss_tts_nano"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
gpt2_cfg = kwargs.pop("gpt2_config", None) or {}
|
||||
if hasattr(gpt2_cfg, "to_dict"):
|
||||
gpt2_cfg = gpt2_cfg.to_dict()
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# --- GPT2 backbone parameters (exposed at top level for vLLM) ---
|
||||
self.hidden_size = gpt2_cfg.get("n_embd", 768)
|
||||
self.num_hidden_layers = gpt2_cfg.get("n_layer", 12)
|
||||
self.num_attention_heads = gpt2_cfg.get("n_head", 12)
|
||||
self.num_key_value_heads = self.num_attention_heads # no GQA in GPT2
|
||||
self.head_dim = self.hidden_size // self.num_attention_heads
|
||||
self.vocab_size = gpt2_cfg.get("vocab_size", 16384)
|
||||
self.max_position_embeddings = gpt2_cfg.get("n_positions", 32768)
|
||||
self.intermediate_size = gpt2_cfg.get("n_inner", self.hidden_size * 4)
|
||||
|
||||
# --- Audio codec parameters ---
|
||||
self.n_vq: int = getattr(self, "n_vq", 16)
|
||||
self.audio_vocab_size: int = getattr(self, "audio_vocab_size", 1024)
|
||||
self.audio_start_token_id: int = getattr(self, "audio_start_token_id", 6)
|
||||
self.audio_end_token_id: int = getattr(self, "audio_end_token_id", 7)
|
||||
self.audio_user_slot_token_id: int = getattr(self, "audio_user_slot_token_id", 8)
|
||||
self.audio_assistant_slot_token_id: int = getattr(self, "audio_assistant_slot_token_id", 9)
|
||||
self.audio_tokenizer_pretrained_name_or_path: str = getattr(
|
||||
self,
|
||||
"audio_tokenizer_pretrained_name_or_path",
|
||||
"OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano",
|
||||
)
|
||||
self.audio_tokenizer_sample_rate: int = getattr(self, "audio_tokenizer_sample_rate", 48000)
|
||||
|
||||
# vLLM requires speculative_config to be absent or None
|
||||
self.speculative_config = None
|
||||
|
||||
def get_text_config(self, **kwargs):
|
||||
"""Return self so vLLM uses our top-level config."""
|
||||
return self
|
||||
@@ -0,0 +1,361 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MOSS-TTS-Nano single-stage model for vLLM-Omni.
|
||||
|
||||
Runs in a single GPUGenerationWorker stage. The 0.1B AR LM and the
|
||||
MOSS-Audio-Tokenizer-Nano codec are both loaded here; the full
|
||||
text → audio-codes → waveform pipeline executes inside forward().
|
||||
|
||||
Streaming (realtime) generation is supported via model.inference_stream():
|
||||
audio chunks are collected as the LM emits tokens and returned as a
|
||||
concatenated waveform. Future work can expose per-chunk streaming via
|
||||
vLLM-Omni's SSE audio endpoint.
|
||||
|
||||
Weight loading deliberately happens inside load_weights() -- NOT __init__ --
|
||||
so that vLLM initialises distributed state before any CUDA allocations occur.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Default sampling parameters matching the upstream demo defaults.
|
||||
_DEFAULT_TEXT_TEMPERATURE = 1.0
|
||||
_DEFAULT_TEXT_TOP_P = 1.0
|
||||
_DEFAULT_TEXT_TOP_K = 50
|
||||
_DEFAULT_AUDIO_TEMPERATURE = 0.8
|
||||
_DEFAULT_AUDIO_TOP_P = 0.95
|
||||
_DEFAULT_AUDIO_TOP_K = 25
|
||||
_DEFAULT_AUDIO_REPETITION_PENALTY = 1.2
|
||||
_DEFAULT_MAX_NEW_FRAMES = 375
|
||||
_DEFAULT_VOICE = "Junhao"
|
||||
_DEFAULT_MODE = "voice_clone"
|
||||
|
||||
|
||||
def _pick(info: dict, key: str, default):
|
||||
"""Extract scalar from additional_information dict (list or plain value)."""
|
||||
val = info.get(key, default)
|
||||
if isinstance(val, (list, tuple)) and len(val) > 0:
|
||||
return val[0]
|
||||
return val if val is not None else default
|
||||
|
||||
|
||||
class MossTTSNanoForGeneration(nn.Module):
|
||||
"""Single-stage MOSS-TTS-Nano generation model.
|
||||
|
||||
Integrates the 0.1B MOSS-TTS-Nano LM and the MOSS-Audio-Tokenizer-Nano
|
||||
codec into a single vLLM-Omni generation stage (GPUGenerationWorker).
|
||||
|
||||
The model uses trust_remote_code=True to load the upstream HuggingFace
|
||||
model classes, keeping our implementation thin and easy to update as
|
||||
the upstream model evolves.
|
||||
|
||||
Supported request fields in additional_information:
|
||||
text (str) – text to synthesize [required]
|
||||
voice (str) – built-in voice preset name, default "Junhao"
|
||||
mode (str) – "voice_clone" (default) or "continuation"
|
||||
prompt_audio_path (str) – path to reference WAV/MP3 for custom voice clone
|
||||
prompt_text (str) – reference transcript (for continuation mode)
|
||||
max_new_frames (int) – max AR frames, default 375 (~14s audio)
|
||||
text_temperature (float) – LM text-layer temperature
|
||||
text_top_p (float) – LM text-layer top-p
|
||||
text_top_k (int) – LM text-layer top-k
|
||||
audio_temperature (float) – LM audio-layer temperature
|
||||
audio_top_p (float) – LM audio-layer top-p
|
||||
audio_top_k (int) – LM audio-layer top-k
|
||||
audio_repetition_penalty (float) – LM audio repetition penalty
|
||||
seed (int) – optional random seed for reproducibility
|
||||
"""
|
||||
|
||||
requires_raw_input_tokens = True
|
||||
have_multimodal_outputs = True
|
||||
has_preprocess = False
|
||||
has_postprocess = False
|
||||
enable_update_additional_information = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.model_path: str = vllm_config.model_config.model
|
||||
|
||||
# Populated by load_weights(); kept None until then so __init__ is
|
||||
# allocation-free (vLLM may call __init__ before CUDA is ready).
|
||||
self._lm: nn.Module | None = None
|
||||
self._audio_tokenizer: nn.Module | None = None
|
||||
self._device: torch.device | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Dummy parameter so vLLM can resolve the device via next(parameters()).
|
||||
self._sentinel = nn.Parameter(torch.zeros(1), requires_grad=False)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load the MOSS-TTS-Nano LM and MOSS-Audio-Tokenizer-Nano codec.
|
||||
|
||||
Both models are loaded via trust_remote_code so we stay in sync
|
||||
with any upstream architecture changes without reimplementing
|
||||
the model code ourselves.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._lm is not None:
|
||||
return set()
|
||||
|
||||
try:
|
||||
device = next(self.parameters()).device
|
||||
except StopIteration:
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
self._device = device
|
||||
|
||||
# Select compute dtype: bfloat16 on capable CUDA, float32 elsewhere.
|
||||
if device.type == "cuda" and torch.cuda.is_bf16_supported():
|
||||
tts_dtype = torch.bfloat16
|
||||
elif device.type == "cuda":
|
||||
tts_dtype = torch.float16
|
||||
else:
|
||||
tts_dtype = torch.float32
|
||||
|
||||
# --- Load the AR language model ---
|
||||
logger.info("Loading MOSS-TTS-Nano LM from %s (dtype=%s)", self.model_path, tts_dtype)
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
lm = AutoModelForCausalLM.from_pretrained(
|
||||
self.model_path,
|
||||
trust_remote_code=True,
|
||||
torch_dtype=tts_dtype,
|
||||
)
|
||||
if device.type == "cuda":
|
||||
# Use SDPA when flash_attention_2 is unavailable.
|
||||
try:
|
||||
lm._set_attention_implementation("flash_attention_2")
|
||||
except Exception:
|
||||
lm._set_attention_implementation("sdpa")
|
||||
lm.to(device=device)
|
||||
lm.eval()
|
||||
self._lm = lm
|
||||
logger.info("MOSS-TTS-Nano LM loaded on %s", device)
|
||||
|
||||
# --- Load the audio tokenizer (codec) ---
|
||||
codec_path: str = getattr(
|
||||
self.config,
|
||||
"audio_tokenizer_pretrained_name_or_path",
|
||||
"OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano",
|
||||
)
|
||||
logger.info("Loading MOSS-Audio-Tokenizer-Nano from %s", codec_path)
|
||||
from transformers import AutoModel
|
||||
|
||||
audio_tokenizer = AutoModel.from_pretrained(
|
||||
codec_path,
|
||||
trust_remote_code=True,
|
||||
torch_dtype=torch.float32, # codec runs in fp32 for stability
|
||||
)
|
||||
audio_tokenizer.to(device=device)
|
||||
audio_tokenizer.eval()
|
||||
self._audio_tokenizer = audio_tokenizer
|
||||
logger.info("MOSS-Audio-Tokenizer-Nano loaded on %s", device)
|
||||
|
||||
# Consume the vLLM weight iterator (we loaded weights ourselves above).
|
||||
for _ in weights:
|
||||
pass
|
||||
return set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Dummy run support (vLLM profiling / KV-cache estimation)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict]:
|
||||
return [{"text": "hello", "voice": _DEFAULT_VOICE, "_is_dummy": True}] * num_reqs
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core forward pass
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
"""Run full MOSS-TTS-Nano pipeline: text → audio codes → waveform.
|
||||
|
||||
Each request in the batch is processed independently. Results are
|
||||
returned as lists so the generation runner can demultiplex them.
|
||||
"""
|
||||
sr = getattr(self.config, "audio_tokenizer_sample_rate", 48000)
|
||||
sr_tensor = torch.tensor(sr, dtype=torch.int32)
|
||||
empty = torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
if not runtime_additional_information:
|
||||
# Profiling / dummy run — return empty audio.
|
||||
n = 1 if input_ids is None else max(1, input_ids.shape[0])
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": [empty] * n,
|
||||
"sr": [sr_tensor] * n,
|
||||
},
|
||||
)
|
||||
|
||||
if self._lm is None or self._audio_tokenizer is None:
|
||||
raise RuntimeError("MOSS-TTS-Nano model not loaded. Was load_weights() called?")
|
||||
|
||||
waveforms: list[torch.Tensor] = []
|
||||
srs: list[torch.Tensor] = []
|
||||
|
||||
for info in runtime_additional_information:
|
||||
if info.get("_is_dummy"):
|
||||
waveforms.append(empty)
|
||||
srs.append(sr_tensor)
|
||||
continue
|
||||
|
||||
waveform = self._run_single_inference(info, sr)
|
||||
waveforms.append(waveform)
|
||||
srs.append(sr_tensor)
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": waveforms, "sr": srs},
|
||||
)
|
||||
|
||||
def _run_single_inference(self, info: dict[str, Any], sr: int) -> torch.Tensor:
|
||||
"""Run inference for a single request and return the waveform tensor."""
|
||||
text: str = str(_pick(info, "text", "") or "")
|
||||
if not text.strip():
|
||||
logger.warning("MOSS-TTS-Nano received empty text; returning silence.")
|
||||
return torch.zeros((sr,), dtype=torch.float32)
|
||||
|
||||
voice: str = str(_pick(info, "voice", _DEFAULT_VOICE))
|
||||
mode: str = str(_pick(info, "mode", _DEFAULT_MODE))
|
||||
prompt_audio_path: str | None = _pick(info, "prompt_audio_path", None)
|
||||
if prompt_audio_path is not None:
|
||||
prompt_audio_path = str(prompt_audio_path)
|
||||
prompt_text: str | None = _pick(info, "prompt_text", None)
|
||||
if prompt_text is not None:
|
||||
prompt_text = str(prompt_text)
|
||||
max_new_frames: int = int(_pick(info, "max_new_frames", _DEFAULT_MAX_NEW_FRAMES))
|
||||
seed: int | None = _pick(info, "seed", None)
|
||||
if seed is not None:
|
||||
seed = int(seed)
|
||||
|
||||
sampling = {
|
||||
"text_temperature": float(_pick(info, "text_temperature", _DEFAULT_TEXT_TEMPERATURE)),
|
||||
"text_top_p": float(_pick(info, "text_top_p", _DEFAULT_TEXT_TOP_P)),
|
||||
"text_top_k": int(_pick(info, "text_top_k", _DEFAULT_TEXT_TOP_K)),
|
||||
"audio_temperature": float(_pick(info, "audio_temperature", _DEFAULT_AUDIO_TEMPERATURE)),
|
||||
"audio_top_p": float(_pick(info, "audio_top_p", _DEFAULT_AUDIO_TOP_P)),
|
||||
"audio_top_k": int(_pick(info, "audio_top_k", _DEFAULT_AUDIO_TOP_K)),
|
||||
"audio_repetition_penalty": float(
|
||||
_pick(info, "audio_repetition_penalty", _DEFAULT_AUDIO_REPETITION_PENALTY)
|
||||
),
|
||||
}
|
||||
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
device = self._device or torch.device("cpu")
|
||||
|
||||
# Use a temp file as the output path (the upstream API requires a path).
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
output_path = tmp.name
|
||||
|
||||
audio_chunks: list[torch.Tensor] = []
|
||||
|
||||
try:
|
||||
for event in self._lm.inference_stream(
|
||||
text=text,
|
||||
output_audio_path=output_path,
|
||||
mode=mode,
|
||||
prompt_text=prompt_text,
|
||||
prompt_audio_path=prompt_audio_path,
|
||||
text_tokenizer_path=self.model_path,
|
||||
audio_tokenizer=self._audio_tokenizer,
|
||||
device=device,
|
||||
nq=None,
|
||||
max_new_frames=max_new_frames,
|
||||
do_sample=True,
|
||||
use_kv_cache=True,
|
||||
**sampling,
|
||||
):
|
||||
event_type = str(event.get("type", ""))
|
||||
if event_type == "audio":
|
||||
waveform = event.get("waveform")
|
||||
if waveform is not None:
|
||||
chunk = torch.as_tensor(waveform, dtype=torch.float32).cpu()
|
||||
audio_chunks.append(chunk)
|
||||
elif event_type == "result":
|
||||
# Final event -- prefer the full waveform from the result
|
||||
# if no intermediate chunks were collected.
|
||||
if not audio_chunks:
|
||||
waveform = event.get("waveform")
|
||||
if waveform is not None:
|
||||
audio_chunks.append(
|
||||
torch.as_tensor(waveform, dtype=torch.float32).cpu()
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("MOSS-TTS-Nano inference failed for text=%r", text[:80])
|
||||
return torch.zeros((sr,), dtype=torch.float32)
|
||||
finally:
|
||||
# Clean up temp file.
|
||||
try:
|
||||
Path(output_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not audio_chunks:
|
||||
logger.warning("MOSS-TTS-Nano produced no audio for text=%r", text[:80])
|
||||
return torch.zeros((sr,), dtype=torch.float32)
|
||||
|
||||
# Concatenate all streaming chunks into a single waveform.
|
||||
# inference_stream yields chunks that are 1D or 2D [samples] / [channels, samples].
|
||||
# Flatten to 1D for the vLLM-Omni audio output format.
|
||||
processed: list[torch.Tensor] = []
|
||||
for chunk in audio_chunks:
|
||||
if chunk.ndim == 2:
|
||||
# [channels, samples] → interleave to [samples * channels] for stereo
|
||||
processed.append(chunk.T.reshape(-1))
|
||||
else:
|
||||
processed.append(chunk.reshape(-1))
|
||||
|
||||
return torch.cat(processed, dim=0)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# vLLM boilerplate: embed_input_ids required by generation runner
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
# Generation stage does not use token embeddings -- return a dummy
|
||||
# zero tensor of the expected shape so the runner's prefill works.
|
||||
hidden = int(getattr(self.config, "hidden_size", 768))
|
||||
return torch.zeros(
|
||||
(input_ids.shape[0], hidden),
|
||||
device=input_ids.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
@@ -163,6 +163,12 @@ _OMNI_MODELS = {
|
||||
"VoxtralTTSAudioGenerationForConditionalGeneration",
|
||||
),
|
||||
"VoxtralTTSAudioTokenizer": ("voxtral_tts", "voxtral_tts_audio_tokenizer", "VoxtralTTSAudioTokenizer"),
|
||||
## MOSS-TTS-Nano
|
||||
"MossTTSNanoForCausalLM": (
|
||||
"moss_tts_nano",
|
||||
"moss_tts_nano",
|
||||
"MossTTSNanoForGeneration",
|
||||
),
|
||||
"DyninOmniForConditionalGeneration": (
|
||||
"dynin_omni",
|
||||
"dynin_omni",
|
||||
|
||||
Reference in new issue
Block a user