[Model] Add Voxtral TTS model (#1803)
Signed-off-by: Chen-Yo Sun <chenyo.sun@mistral.ai>
This commit is contained in:
1 parent
b766c4702c
commit
e9fe78e28c
29 files changed
+5135
-51
No files matched your search
@@ -140,6 +140,18 @@ _OMNI_MODELS = {
|
||||
"fish_speech_dac_decoder",
|
||||
"FishSpeechDACDecoder",
|
||||
),
|
||||
## Voxtral TTS
|
||||
"VoxtralTTSForConditionalGeneration": (
|
||||
"voxtral_tts",
|
||||
"voxtral_tts",
|
||||
"VoxtralTTSForConditionalGeneration",
|
||||
),
|
||||
"VoxtralTTSAudioGeneration": (
|
||||
"voxtral_tts",
|
||||
"voxtral_tts_audio_generation",
|
||||
"VoxtralTTSAudioGenerationForConditionalGeneration",
|
||||
),
|
||||
"VoxtralTTSAudioTokenizer": ("voxtral_tts", "voxtral_tts_audio_tokenizer", "VoxtralTTSAudioTokenizer"),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import MistralConfigParser, register_config_parser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VoxtralTTSConfig(PretrainedConfig):
|
||||
"""HuggingFace-style config for Voxtral TTS models."""
|
||||
|
||||
model_type = "voxtral_tts"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: PretrainedConfig | dict | None = None,
|
||||
audio_config: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(text_config, PretrainedConfig):
|
||||
self.text_config = text_config
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = PretrainedConfig.from_dict(text_config)
|
||||
else:
|
||||
self.text_config = PretrainedConfig()
|
||||
|
||||
self.audio_config = audio_config or {}
|
||||
|
||||
def get_text_config(self, **kwargs: Any) -> PretrainedConfig:
|
||||
return self.text_config
|
||||
|
||||
|
||||
@register_config_parser("mistral")
|
||||
class VoxtralTTSConfigParser(MistralConfigParser):
|
||||
"""Config parser that extends the base Mistral parser with TTS support.
|
||||
|
||||
This only support voxtral_tts for now.
|
||||
"""
|
||||
|
||||
def _remap_mistral_audio_args(self, config_dict: dict) -> dict:
|
||||
encoder_args = config_dict["multimodal"].pop("audio_model_args")
|
||||
audio_tokenizer_args = config_dict["multimodal"].pop("audio_tokenizer_args", None)
|
||||
audio_config = {}
|
||||
if encoder_args is not None:
|
||||
audio_config = {
|
||||
"sampling_rate": encoder_args["audio_encoding_args"]["sampling_rate"],
|
||||
"codec_args": audio_tokenizer_args,
|
||||
"audio_model_args": encoder_args,
|
||||
"speaker_id": audio_tokenizer_args.get("voice", {}),
|
||||
}
|
||||
return audio_config
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model: str | Path,
|
||||
trust_remote_code: bool,
|
||||
revision: str | None = None,
|
||||
code_revision: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[dict, PretrainedConfig]:
|
||||
from vllm.transformers_utils.config import (
|
||||
_download_mistral_config_file,
|
||||
)
|
||||
|
||||
config_dict = _download_mistral_config_file(model, revision)
|
||||
|
||||
from vllm.transformers_utils.configs.mistral import (
|
||||
_remap_general_mistral_args,
|
||||
_remap_mistral_quantization_args,
|
||||
)
|
||||
|
||||
# Extract audio config before building text config
|
||||
audio_config = {}
|
||||
if (config_dict.get("multimodal") or {}).get("audio_model_args"):
|
||||
audio_config = self._remap_mistral_audio_args(config_dict)
|
||||
|
||||
# Build text_config from the top-level keys
|
||||
non_text_keys = {"multimodal"}
|
||||
text_config = {k: v for k, v in config_dict.items() if k not in non_text_keys}
|
||||
text_config = _remap_general_mistral_args(text_config)
|
||||
if text_config.get("quantization"):
|
||||
text_config = _remap_mistral_quantization_args(text_config)
|
||||
|
||||
# The text sub-model is a plain MistralForCausalLM
|
||||
text_config.setdefault("architectures", ["MistralForCausalLM"])
|
||||
|
||||
config = VoxtralTTSConfig(
|
||||
text_config=PretrainedConfig.from_dict(text_config),
|
||||
audio_config=audio_config,
|
||||
architectures=config_dict.get("architectures", ["VoxtralTTSForConditionalGeneration"]),
|
||||
)
|
||||
|
||||
return config_dict, config
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
CUDA Graph wrapper for AcousticTransformer in VoxtralTTS.
|
||||
|
||||
Currently it support FlowMatching implementation only.
|
||||
|
||||
Captures the AcousticTransformer forward pass (semantic logit +
|
||||
n-step Euler ODE with CFG) into CUDA graphs for fixed batch sizes,
|
||||
eliminating kernel launch overhead on every decode step.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch.cuda import CUDAGraph
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
|
||||
AudioSpecialTokens,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CUDAGraphAcousticTransformerWrapper:
|
||||
"""
|
||||
CUDA Graph wrapper for the acoustic transformer.
|
||||
|
||||
Replaces the vLLM Sampler path (which has Python-level branching and dynamic
|
||||
tensor allocation) with a CUDA-graph-compatible path using torch.argmax
|
||||
(equivalent to top_k=1 greedy sampling) and pre-allocated buffers.
|
||||
"""
|
||||
|
||||
DEFAULT_CAPTURE_SIZES = [1, 2, 4, 8, 16, 32]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model,
|
||||
capture_sizes: list[int] | None = None,
|
||||
):
|
||||
self.model = model
|
||||
self.acoustic_transformer = model.acoustic_transformer
|
||||
|
||||
self.capture_sizes = sorted(capture_sizes or self.DEFAULT_CAPTURE_SIZES)
|
||||
|
||||
# Pre-compute constants from the acoustic transformer
|
||||
self.empty_audio_token_id = AudioSpecialTokens.id(AudioSpecialTokens.empty_audio)
|
||||
self.end_audio_token_id = AudioSpecialTokens.id(AudioSpecialTokens.end_audio)
|
||||
self.semantic_mask_start = len(AudioSpecialTokens) + self.acoustic_transformer.model_args.semantic_codebook_size
|
||||
self.n_acoustic_codebook = self.acoustic_transformer.model_args.n_acoustic_codebook
|
||||
self.acoustic_embeddings_levels = self.acoustic_transformer.acoustic_embeddings_levels
|
||||
|
||||
self.cfg_alpha = 1.2
|
||||
self.n_steps = 16
|
||||
|
||||
# Graph storage
|
||||
self.graphs: dict[int, CUDAGraph] = {}
|
||||
self.static_inputs: dict[int, torch.Tensor] = {}
|
||||
self.static_noise: dict[int, torch.Tensor] = {}
|
||||
self.static_fake_eos: dict[int, torch.Tensor] = {}
|
||||
self.static_audio_codes: dict[int, torch.Tensor] = {}
|
||||
|
||||
self.enabled = False
|
||||
self._warmed_up = False
|
||||
|
||||
def _warmup_and_capture(self, device: torch.device, dtype: torch.dtype, hidden_dim: int):
|
||||
"""Perform eager warmup and CUDA graph capture for all bucket sizes."""
|
||||
if self._warmed_up:
|
||||
logger.warning("CUDAGraphAcousticTransformerWrapper already warmed up, skipping")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"CUDAGraphAcousticTransformerWrapper: starting warmup and capture for sizes %s",
|
||||
self.capture_sizes,
|
||||
)
|
||||
|
||||
# Pre-create persistent buffers
|
||||
self.timesteps = torch.linspace(0, 1, self.n_steps, device=device, dtype=dtype)
|
||||
self.fake_eos_one = torch.tensor(1.0, dtype=dtype, device=device)
|
||||
self.fake_eos_zero = torch.tensor(0.0, dtype=dtype, device=device)
|
||||
|
||||
# Phase 1: Eager warmup for ALL capture sizes
|
||||
for size in self.capture_sizes:
|
||||
dummy = torch.zeros(size, hidden_dim, device=device, dtype=dtype)
|
||||
with torch.no_grad():
|
||||
self._forward_cudagraph_compatible(dummy)
|
||||
|
||||
torch.cuda.synchronize(device)
|
||||
|
||||
# Phase 2: Capture graphs
|
||||
for size in self.capture_sizes:
|
||||
try:
|
||||
self._capture_graph_for_size(size, device, dtype, hidden_dim)
|
||||
logger.info(" Captured CUDA Graph for batch_size=%d", size)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
" Failed to capture CUDA Graph for batch_size=%d",
|
||||
size,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self.enabled = True
|
||||
self._warmed_up = True
|
||||
logger.info(
|
||||
"CUDAGraphAcousticTransformerWrapper warmup complete. Captured %d/%d graphs.",
|
||||
len(self.graphs),
|
||||
len(self.capture_sizes),
|
||||
)
|
||||
|
||||
def _forward_cudagraph_compatible(self, hidden_states: torch.Tensor, noise: torch.Tensor | None = None):
|
||||
"""
|
||||
The actual computation captured by the CUDA graph.
|
||||
|
||||
This replaces the full compute_mm_logits -> acoustic_transformer.forward() path
|
||||
with a graph-compatible version:
|
||||
- Uses argmax instead of vLLM Sampler (equivalent for top_k=1)
|
||||
- Uses pre-created timesteps buffer instead of torch.linspace
|
||||
- Uses pre-created scalar tensors for torch.where
|
||||
- Calls _predict_velocity directly
|
||||
- Uses a pre-allocated noise buffer to avoid baking random state
|
||||
into the CUDA graph
|
||||
"""
|
||||
at = self.acoustic_transformer
|
||||
B = hidden_states.shape[0]
|
||||
|
||||
# --- Semantic logits via linear projection ---
|
||||
semantic_logit = at.semantic_codebook_output(hidden_states).float()
|
||||
semantic_logit[:, self.empty_audio_token_id] = -float("inf")
|
||||
semantic_logit[:, self.semantic_mask_start :] = -float("inf")
|
||||
|
||||
# argmax == top_k=1 greedy sampling
|
||||
semantic_code = semantic_logit.argmax(dim=-1, keepdim=True) # (B, 1)
|
||||
|
||||
# --- Flow matching: Euler ODE ---
|
||||
should_decode = semantic_code.squeeze(1) != self.end_audio_token_id
|
||||
|
||||
if noise is not None:
|
||||
x = noise
|
||||
else:
|
||||
x = torch.randn(B, self.n_acoustic_codebook, device=hidden_states.device, dtype=hidden_states.dtype)
|
||||
|
||||
# Pre-compute zero hidden states for unconditional CFG branch
|
||||
hidden_states_zero = torch.zeros_like(hidden_states)
|
||||
|
||||
timesteps = self.timesteps
|
||||
for i in range(len(timesteps) - 1):
|
||||
t = timesteps[i]
|
||||
dt = timesteps[i + 1] - timesteps[i]
|
||||
|
||||
# Batch conditional + unconditional velocity in a single forward pass
|
||||
t_emb = at.time_embedding(t.view(-1, 1).repeat(B, 1)).to(hidden_states.dtype)
|
||||
x_batched = torch.cat([x, x], dim=0) # (2B, C)
|
||||
llm_batched = torch.cat([hidden_states, hidden_states_zero], dim=0) # (2B, D)
|
||||
t_emb_batched = t_emb.repeat(2, 1) # (2B, D)
|
||||
|
||||
v_all = at._predict_velocity(x_t=x_batched, llm_output=llm_batched, t_emb=t_emb_batched)
|
||||
v_t, uncond_v_t = v_all[:B], v_all[B:]
|
||||
|
||||
# CFG combination
|
||||
v_t = self.cfg_alpha * v_t + (1 - self.cfg_alpha) * uncond_v_t
|
||||
|
||||
x = x + v_t * dt
|
||||
|
||||
# --- Quantize ---
|
||||
sampled = torch.clamp(x, -1, 1)
|
||||
scaled_x = ((sampled + 1) / 2) * (self.acoustic_embeddings_levels - 1)
|
||||
output_codes = scaled_x.round().long()
|
||||
output_codes[~should_decode] = self.empty_audio_token_id
|
||||
acoustic_codes = output_codes + len(AudioSpecialTokens)
|
||||
|
||||
# --- Combine semantic + acoustic ---
|
||||
audio_codes = torch.cat([semantic_code, acoustic_codes], dim=1) # (B, 1 + n_acoustic)
|
||||
|
||||
# --- Compute fake_eos ---
|
||||
fake_eos = torch.where(
|
||||
audio_codes[:, 0] == self.end_audio_token_id,
|
||||
self.fake_eos_one,
|
||||
self.fake_eos_zero,
|
||||
)
|
||||
|
||||
return fake_eos, audio_codes
|
||||
|
||||
def _capture_graph_for_size(
|
||||
self,
|
||||
size: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
hidden_dim: int,
|
||||
):
|
||||
"""Capture a CUDA graph for a specific batch size."""
|
||||
static_input = torch.zeros(size, hidden_dim, device=device, dtype=dtype)
|
||||
static_noise = torch.randn(size, self.n_acoustic_codebook, device=device, dtype=dtype)
|
||||
|
||||
# Stabilizing eager run
|
||||
with torch.no_grad():
|
||||
_ = self._forward_cudagraph_compatible(static_input, noise=static_noise)
|
||||
|
||||
torch.cuda.synchronize(device)
|
||||
|
||||
graph = CUDAGraph()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.graph(graph):
|
||||
static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible(
|
||||
static_input, noise=static_noise
|
||||
)
|
||||
|
||||
self.graphs[size] = graph
|
||||
self.static_inputs[size] = static_input
|
||||
self.static_noise[size] = static_noise
|
||||
self.static_fake_eos[size] = static_fake_eos
|
||||
self.static_audio_codes[size] = static_audio_codes
|
||||
|
||||
def _get_padded_size(self, actual_size: int) -> int | None:
|
||||
"""Round up to the nearest captured bucket size."""
|
||||
for size in self.capture_sizes:
|
||||
if actual_size <= size:
|
||||
return size
|
||||
return None
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, dict[str, list[torch.Tensor]] | None]:
|
||||
"""
|
||||
Drop-in replacement for model.compute_mm_logits().
|
||||
|
||||
Falls back to eager execution if:
|
||||
- CUDA graph is not enabled/warmed up
|
||||
- Batch size exceeds largest captured size
|
||||
"""
|
||||
actual_size = hidden_states.shape[0]
|
||||
|
||||
if not self.enabled or not self._warmed_up:
|
||||
return self.model.compute_mm_logits(hidden_states)
|
||||
|
||||
padded_size = self._get_padded_size(actual_size)
|
||||
if padded_size is None or padded_size not in self.graphs:
|
||||
return self.model.compute_mm_logits(hidden_states)
|
||||
|
||||
# Zero static input, then copy actual data
|
||||
self.static_inputs[padded_size].zero_()
|
||||
self.static_inputs[padded_size][:actual_size] = hidden_states
|
||||
|
||||
# Fill noise buffer with fresh random values before replay so the
|
||||
# flow-matching ODE starts from different initial noise each time.
|
||||
self.static_noise[padded_size].normal_()
|
||||
|
||||
# Replay captured graph
|
||||
self.graphs[padded_size].replay()
|
||||
|
||||
# Clone and slice outputs for actual batch size
|
||||
fake_eos = self.static_fake_eos[padded_size][:actual_size].clone()
|
||||
audio_codes = self.static_audio_codes[padded_size][:actual_size].clone()
|
||||
|
||||
# Package into expected format: (fake_eos, {"audio": [list of tensors]})
|
||||
audio_list = list(torch.split(audio_codes.unsqueeze(1), 1, dim=0))
|
||||
mm_tokens = {"audio": audio_list}
|
||||
|
||||
return fake_eos, mm_tokens
|
||||
@@ -0,0 +1,372 @@
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import hf_hub_download
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.voxtral_tts.cuda_graph_acoustic_transformer_wrapper import (
|
||||
CUDAGraphAcousticTransformerWrapper,
|
||||
)
|
||||
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
|
||||
VoxtralTTSDummyInputsBuilder,
|
||||
VoxtralTTSMultiModalProcessor,
|
||||
VoxtralTTSProcessingInfo,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def parse_batched_audio_input(input_ids: torch.Tensor, num_codebooks: int) -> tuple[list[torch.Tensor], list[int]]:
|
||||
"""Parse batched input_ids with [ctx_frames, context_length, ...tokens] format.
|
||||
|
||||
Each request in the batch is laid out as:
|
||||
[ctx_frames, context_length, <flat audio tokens>]
|
||||
where the flat audio tokens have length (ctx_frames + context_length) * num_codebooks.
|
||||
|
||||
Returns:
|
||||
all_audio_tokens: list of (num_frames, num_codebooks) tensors per request.
|
||||
all_ctx_frames: list of ctx_frames values per request.
|
||||
"""
|
||||
all_audio_tokens: list[torch.Tensor] = []
|
||||
all_ctx_frames: list[int] = []
|
||||
offset = 0
|
||||
while offset < input_ids.numel():
|
||||
ctx_frames = int(input_ids[offset].item())
|
||||
context_length = int(input_ids[offset + 1].item())
|
||||
offset += 2
|
||||
|
||||
req_input_ids_length = (ctx_frames + context_length) * num_codebooks
|
||||
req_input_ids = input_ids[offset : offset + req_input_ids_length]
|
||||
offset += req_input_ids_length
|
||||
assert req_input_ids.numel() % num_codebooks == 0, (
|
||||
f"Number of elements must be divisible by {num_codebooks} but get {req_input_ids.numel()=}"
|
||||
)
|
||||
num_of_audios = req_input_ids.numel() // num_codebooks
|
||||
audio_tokens = req_input_ids.view(num_of_audios, num_codebooks)
|
||||
all_audio_tokens.append(audio_tokens)
|
||||
all_ctx_frames.append(ctx_frames)
|
||||
return all_audio_tokens, all_ctx_frames
|
||||
|
||||
|
||||
def apply_ctx_frames_cutting(
|
||||
batch_audio_arrays: list[torch.Tensor],
|
||||
all_ctx_frames: list[int],
|
||||
downsample_factor: int,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Cut leading context samples from decoded audio arrays.
|
||||
|
||||
Args:
|
||||
batch_audio_arrays: raw decoded audio arrays, one per request.
|
||||
all_ctx_frames: number of context frames per request.
|
||||
downsample_factor: samples per frame in the audio tokenizer.
|
||||
|
||||
Returns:
|
||||
List of audio arrays with context samples removed.
|
||||
"""
|
||||
result: list[torch.Tensor] = []
|
||||
for audio_array, ctx_frames in zip(batch_audio_arrays, all_ctx_frames):
|
||||
if ctx_frames > 0:
|
||||
new_sample_cut = downsample_factor * ctx_frames
|
||||
audio_array = audio_array[new_sample_cut:]
|
||||
result.append(audio_array)
|
||||
return result
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
VoxtralTTSMultiModalProcessor,
|
||||
info=VoxtralTTSProcessingInfo,
|
||||
dummy_inputs=VoxtralTTSDummyInputsBuilder,
|
||||
)
|
||||
class VoxtralTTSForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
CustomProcessMixin,
|
||||
):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.config = config
|
||||
self.repo_id = vllm_config.model_config.model
|
||||
self.is_hf_model = not Path(self.repo_id).is_dir()
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
if self.model_stage == "audio_generation":
|
||||
self.has_preprocess = True
|
||||
self.has_postprocess = True
|
||||
self.requires_raw_input_tokens = True
|
||||
self.set_custom_preprocess(self.tts_preprocess)
|
||||
self.set_custom_postprocess(self.tts_postprocess)
|
||||
self.audio_generation = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "audio_generation"),
|
||||
architectures=["VoxtralTTSAudioGeneration"],
|
||||
)
|
||||
self.model = self.audio_generation
|
||||
self.audio_tokenizer = None
|
||||
self._cudagraph_acoustic_transformer = None
|
||||
self._vllm_config = vllm_config
|
||||
tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self._audio_token_id = tokenizer.instruct.audio_encoder.special_ids.audio
|
||||
speaker_id = config.audio_config.get("speaker_id", None)
|
||||
if speaker_id and self.is_hf_model:
|
||||
self.voice_to_embedding = {
|
||||
sid: torch.load(hf_hub_download(repo_id=self.repo_id, filename=f"voice_embedding/{sid}.pt"))
|
||||
for sid in speaker_id
|
||||
}
|
||||
logger.info("Available voice embeddings: %s", list(self.voice_to_embedding.keys()))
|
||||
else:
|
||||
self.voice_to_embedding = {}
|
||||
logger.warning("No speaker_id configured in audio_config.No voice embeddings will be available.")
|
||||
elif self.model_stage == "audio_tokenizer":
|
||||
self.requires_raw_input_tokens = True
|
||||
self.audio_generation = None
|
||||
self.audio_tokenizer = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "audio_tokenizer"),
|
||||
architectures=["VoxtralTTSAudioTokenizer"],
|
||||
)
|
||||
self.model = self.audio_tokenizer
|
||||
else:
|
||||
raise ValueError("Invalid model stage")
|
||||
|
||||
# -------------------- CUDA Graph for acoustic transformer --------------------
|
||||
def _enable_acoustic_transformer_cudagraph(self):
|
||||
"""Initialize and capture CUDA graphs for compute_mm_logits."""
|
||||
if self.model_stage != "audio_generation" or not hasattr(self, "_vllm_config"):
|
||||
return
|
||||
|
||||
model_cfg = getattr(self._vllm_config, "model_config", None)
|
||||
if model_cfg and getattr(model_cfg, "enforce_eager", False):
|
||||
logger.info("CUDA Graph for acoustic transformer not enabled: --enforce-eager is set")
|
||||
return
|
||||
|
||||
try:
|
||||
acoustic_transformer = self.model.acoustic_transformer
|
||||
device = next(acoustic_transformer.parameters()).device
|
||||
dtype = next(acoustic_transformer.parameters()).dtype
|
||||
hidden_dim = self.model.config.text_config.hidden_size
|
||||
|
||||
wrapper = CUDAGraphAcousticTransformerWrapper(self.model)
|
||||
wrapper._warmup_and_capture(device, dtype, hidden_dim)
|
||||
self._cudagraph_acoustic_transformer = wrapper
|
||||
logger.info("CUDA Graph for acoustic transformer enabled")
|
||||
except Exception:
|
||||
logger.warning("Failed to enable CUDA Graph for acoustic transformer", exc_info=True)
|
||||
self._cudagraph_acoustic_transformer = None
|
||||
|
||||
# -------------------- Device utilities --------------------
|
||||
@cached_property
|
||||
def sampler(self):
|
||||
if hasattr(self.model, "sampler"):
|
||||
return self.model.sampler
|
||||
return Sampler()
|
||||
|
||||
def tts_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict | None):
|
||||
self.post_process_idx = 0
|
||||
audio_tokens = info_dict.pop("audio", None)
|
||||
if audio_tokens is not None:
|
||||
kwargs = {"audio_tokens": audio_tokens.to(input_ids.device)}
|
||||
multimodal_embeddings = self.model.embed_multimodal(**kwargs)
|
||||
if input_ids[0] == self._audio_token_id:
|
||||
input_embeds = multimodal_embeddings[0]
|
||||
return input_ids, input_embeds, info_dict
|
||||
voice = info_dict.pop("voice", None)
|
||||
if voice is not None:
|
||||
if isinstance(voice, list):
|
||||
voice = voice[0]
|
||||
multimodal_embeddings = self.voice_to_embedding[voice].to(input_ids.device).clone().detach()
|
||||
is_multimodal = input_ids == self._audio_token_id
|
||||
input_embeds = self.embed_input_ids(
|
||||
input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal
|
||||
)
|
||||
return input_ids, input_embeds, info_dict
|
||||
return input_ids, input_embeds, info_dict
|
||||
|
||||
def tts_postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: object, **info_dict: object | None):
|
||||
update_dict = {}
|
||||
if isinstance(multimodal_outputs, dict) and "audio" in multimodal_outputs:
|
||||
assert self.post_process_idx < len(multimodal_outputs["audio"]), (
|
||||
f"Expect {self.post_process_idx=} < {len(multimodal_outputs['audio'])=}"
|
||||
)
|
||||
update_dict["audio"] = multimodal_outputs["audio"][self.post_process_idx]
|
||||
self.post_process_idx += 1
|
||||
return update_dict
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
if self.model_stage == "audio_tokenizer":
|
||||
return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.config.text_config.hidden_size)
|
||||
return self.model.embed_input_ids(
|
||||
input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
# Delegate to generation model for multimodal processing
|
||||
return self.model.embed_multimodal(**kwargs)
|
||||
|
||||
def last_index_of(self, list, value):
|
||||
return len(list) - 1 - list[::-1].index(value)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors | OmniOutput:
|
||||
if self.model_stage == "audio_generation":
|
||||
if inputs_embeds is not None:
|
||||
input_ids = None
|
||||
hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
if self.model_stage == "audio_tokenizer":
|
||||
if (input_ids == 0).all():
|
||||
logger.info("audio_tokenizer: sample run with dummy input")
|
||||
# TODO(chenyo): Move this to dummy_inputs creation
|
||||
num_codebooks = self.audio_tokenizer.num_codebooks
|
||||
audio_tokens = torch.randint(low=2, high=100, size=(116, num_codebooks), dtype=torch.int32)
|
||||
audio_tokens[-1, :] = 0
|
||||
audio_tokens[-1, 0] = 1
|
||||
batch_audio_arrays = self.audio_tokenizer.decode_helper_batch_async([audio_tokens])
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"audio": batch_audio_arrays},
|
||||
)
|
||||
all_audio_tokens, all_ctx_frames = parse_batched_audio_input(
|
||||
input_ids, num_codebooks=self.audio_tokenizer.num_codebooks
|
||||
)
|
||||
|
||||
# Batch decode all requests at once
|
||||
batch_audio_arrays_raw = self.audio_tokenizer.decode_helper_batch_async(all_audio_tokens)
|
||||
batch_audio_arrays = apply_ctx_frames_cutting(
|
||||
batch_audio_arrays_raw, all_ctx_frames, self.audio_tokenizer.downsample_factor
|
||||
)
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"audio": batch_audio_arrays},
|
||||
)
|
||||
|
||||
def make_omni_output(
|
||||
self, model_outputs: torch.Tensor | OmniOutput | tuple, logits_index: int | None = None, **kwargs
|
||||
) -> OmniOutput:
|
||||
if isinstance(model_outputs, torch.Tensor):
|
||||
if self.model_stage == "audio_generation":
|
||||
hidden_states = model_outputs
|
||||
assert logits_index is not None
|
||||
input_hidden_states = hidden_states[logits_index]
|
||||
if self._cudagraph_acoustic_transformer is not None:
|
||||
fake_eos, multimodal_outputs = self._cudagraph_acoustic_transformer(input_hidden_states)
|
||||
else:
|
||||
fake_eos, multimodal_outputs = self.model.compute_mm_logits(input_hidden_states)
|
||||
hidden_states[logits_index, 0] = fake_eos
|
||||
return OmniOutput(
|
||||
text_hidden_states=hidden_states,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported {self.model_stage} for model_outputs type: {type(model_outputs)}")
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
if self.model_stage == "audio_tokenizer":
|
||||
return model_outputs
|
||||
|
||||
raise ValueError(f"Unsupported {self.model_stage} for model_outputs type: {type(model_outputs)}")
|
||||
|
||||
raise ValueError(f"Unsupported model_outputs type: {type(model_outputs)}")
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
|
||||
# Handle OmniOutput type
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if self.model_stage == "audio_generation":
|
||||
text_logits = self.model.fake_logits_for_audio_tokens(fake_eos=hidden_states)
|
||||
return text_logits
|
||||
if self.model_stage == "audio_tokenizer":
|
||||
raise ValueError("Invalid model stage")
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> SamplerOutput | None:
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loaded_weights = set()
|
||||
remapping_rules = [
|
||||
(r"^audio_tokenizer\.(.*)$", r"\1"), # noqa: E501
|
||||
(
|
||||
r"^mm_audio_embeddings\.audio_codebook_embeddings\.embeddings\.(weight|bias)",
|
||||
r"audio_token_embedding.embeddings.\1",
|
||||
), # noqa: E501
|
||||
(r"^mm_audio_embeddings\.tok_embeddings\.weight", r"tok_embeddings.weight"), # noqa: E501
|
||||
]
|
||||
|
||||
def llm_weights_generator():
|
||||
nonlocal loaded_weights
|
||||
for name, w in weights:
|
||||
is_audio_tokenizer = name.startswith(
|
||||
"mm_audio_embeddings.audio_codebook_embeddings"
|
||||
) or name.startswith("audio_tokenizer.")
|
||||
|
||||
if is_audio_tokenizer and self.audio_tokenizer is not None:
|
||||
# Remap name only when loading audio_tokenizer (Stage-1).
|
||||
# Yield the original name in Stage-0 so audio_generation.load_weights()
|
||||
# can still filter by prefix using its own remapping rules.
|
||||
remapped = name
|
||||
for pattern, repl in remapping_rules:
|
||||
if re.fullmatch(pattern, remapped):
|
||||
remapped = re.sub(pattern, repl, remapped)
|
||||
name = self.audio_tokenizer.load_weight((remapped, w))
|
||||
loaded_weights.add(f"audio_tokenizer.{name}")
|
||||
continue
|
||||
|
||||
yield (name, w)
|
||||
|
||||
audio_generation_weights = list(llm_weights_generator())
|
||||
if self.audio_generation is not None:
|
||||
for name in self.audio_generation.load_weights(audio_generation_weights):
|
||||
loaded_weights.add(f"audio_generation.{name}")
|
||||
|
||||
# If encoder weights were not in the checkpoint, mark them as
|
||||
# "loaded" so the weight-validation does not fail.
|
||||
# encode_waveforms() will raise at runtime if called without encoder weights.
|
||||
if self.audio_tokenizer is not None and not self.audio_tokenizer._encoder_loaded:
|
||||
for name, _ in self.audio_tokenizer.named_parameters():
|
||||
if name.startswith(self.audio_tokenizer._encoder_weight_prefixes):
|
||||
loaded_weights.add(f"audio_tokenizer.{name}")
|
||||
|
||||
# Capture CUDA graphs for compute_mm_logits after weights are loaded
|
||||
self._enable_acoustic_transformer_cudagraph()
|
||||
|
||||
return loaded_weights
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user