[Model] Add Voxtral TTS model (#1803)

Signed-off-by: Chen-Yo Sun <chenyo.sun@mistral.ai>
This commit is contained in:
Chen-Yo Sun authored and GitHub committed 2026-03-19 16:02:11 -04:00
1 parent b766c4702c
commit e9fe78e28c
29 files changed
+5135 -51

No files matched your search

@@ -140,6 +140,18 @@ _OMNI_MODELS = {
"fish_speech_dac_decoder",
"FishSpeechDACDecoder",
),
## Voxtral TTS
"VoxtralTTSForConditionalGeneration": (
"voxtral_tts",
"voxtral_tts",
"VoxtralTTSForConditionalGeneration",
),
"VoxtralTTSAudioGeneration": (
"voxtral_tts",
"voxtral_tts_audio_generation",
"VoxtralTTSAudioGenerationForConditionalGeneration",
),
"VoxtralTTSAudioTokenizer": ("voxtral_tts", "voxtral_tts_audio_tokenizer", "VoxtralTTSAudioTokenizer"),
}
@@ -0,0 +1,99 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.transformers_utils.config import MistralConfigParser, register_config_parser
logger = init_logger(__name__)
class VoxtralTTSConfig(PretrainedConfig):
"""HuggingFace-style config for Voxtral TTS models."""
model_type = "voxtral_tts"
def __init__(
self,
text_config: PretrainedConfig | dict | None = None,
audio_config: dict[str, Any] | None = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
if isinstance(text_config, PretrainedConfig):
self.text_config = text_config
elif isinstance(text_config, dict):
self.text_config = PretrainedConfig.from_dict(text_config)
else:
self.text_config = PretrainedConfig()
self.audio_config = audio_config or {}
def get_text_config(self, **kwargs: Any) -> PretrainedConfig:
return self.text_config
@register_config_parser("mistral")
class VoxtralTTSConfigParser(MistralConfigParser):
"""Config parser that extends the base Mistral parser with TTS support.
This only support voxtral_tts for now.
"""
def _remap_mistral_audio_args(self, config_dict: dict) -> dict:
encoder_args = config_dict["multimodal"].pop("audio_model_args")
audio_tokenizer_args = config_dict["multimodal"].pop("audio_tokenizer_args", None)
audio_config = {}
if encoder_args is not None:
audio_config = {
"sampling_rate": encoder_args["audio_encoding_args"]["sampling_rate"],
"codec_args": audio_tokenizer_args,
"audio_model_args": encoder_args,
"speaker_id": audio_tokenizer_args.get("voice", {}),
}
return audio_config
def parse(
self,
model: str | Path,
trust_remote_code: bool,
revision: str | None = None,
code_revision: str | None = None,
**kwargs: Any,
) -> tuple[dict, PretrainedConfig]:
from vllm.transformers_utils.config import (
_download_mistral_config_file,
)
config_dict = _download_mistral_config_file(model, revision)
from vllm.transformers_utils.configs.mistral import (
_remap_general_mistral_args,
_remap_mistral_quantization_args,
)
# Extract audio config before building text config
audio_config = {}
if (config_dict.get("multimodal") or {}).get("audio_model_args"):
audio_config = self._remap_mistral_audio_args(config_dict)
# Build text_config from the top-level keys
non_text_keys = {"multimodal"}
text_config = {k: v for k, v in config_dict.items() if k not in non_text_keys}
text_config = _remap_general_mistral_args(text_config)
if text_config.get("quantization"):
text_config = _remap_mistral_quantization_args(text_config)
# The text sub-model is a plain MistralForCausalLM
text_config.setdefault("architectures", ["MistralForCausalLM"])
config = VoxtralTTSConfig(
text_config=PretrainedConfig.from_dict(text_config),
audio_config=audio_config,
architectures=config_dict.get("architectures", ["VoxtralTTSForConditionalGeneration"]),
)
return config_dict, config
@@ -0,0 +1,256 @@
"""
CUDA Graph wrapper for AcousticTransformer in VoxtralTTS.
Currently it support FlowMatching implementation only.
Captures the AcousticTransformer forward pass (semantic logit +
n-step Euler ODE with CFG) into CUDA graphs for fixed batch sizes,
eliminating kernel launch overhead on every decode step.
"""
import torch
from torch.cuda import CUDAGraph
from vllm.logger import init_logger
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
AudioSpecialTokens,
)
logger = init_logger(__name__)
class CUDAGraphAcousticTransformerWrapper:
"""
CUDA Graph wrapper for the acoustic transformer.
Replaces the vLLM Sampler path (which has Python-level branching and dynamic
tensor allocation) with a CUDA-graph-compatible path using torch.argmax
(equivalent to top_k=1 greedy sampling) and pre-allocated buffers.
"""
DEFAULT_CAPTURE_SIZES = [1, 2, 4, 8, 16, 32]
def __init__(
self,
model,
capture_sizes: list[int] | None = None,
):
self.model = model
self.acoustic_transformer = model.acoustic_transformer
self.capture_sizes = sorted(capture_sizes or self.DEFAULT_CAPTURE_SIZES)
# Pre-compute constants from the acoustic transformer
self.empty_audio_token_id = AudioSpecialTokens.id(AudioSpecialTokens.empty_audio)
self.end_audio_token_id = AudioSpecialTokens.id(AudioSpecialTokens.end_audio)
self.semantic_mask_start = len(AudioSpecialTokens) + self.acoustic_transformer.model_args.semantic_codebook_size
self.n_acoustic_codebook = self.acoustic_transformer.model_args.n_acoustic_codebook
self.acoustic_embeddings_levels = self.acoustic_transformer.acoustic_embeddings_levels
self.cfg_alpha = 1.2
self.n_steps = 16
# Graph storage
self.graphs: dict[int, CUDAGraph] = {}
self.static_inputs: dict[int, torch.Tensor] = {}
self.static_noise: dict[int, torch.Tensor] = {}
self.static_fake_eos: dict[int, torch.Tensor] = {}
self.static_audio_codes: dict[int, torch.Tensor] = {}
self.enabled = False
self._warmed_up = False
def _warmup_and_capture(self, device: torch.device, dtype: torch.dtype, hidden_dim: int):
"""Perform eager warmup and CUDA graph capture for all bucket sizes."""
if self._warmed_up:
logger.warning("CUDAGraphAcousticTransformerWrapper already warmed up, skipping")
return
logger.info(
"CUDAGraphAcousticTransformerWrapper: starting warmup and capture for sizes %s",
self.capture_sizes,
)
# Pre-create persistent buffers
self.timesteps = torch.linspace(0, 1, self.n_steps, device=device, dtype=dtype)
self.fake_eos_one = torch.tensor(1.0, dtype=dtype, device=device)
self.fake_eos_zero = torch.tensor(0.0, dtype=dtype, device=device)
# Phase 1: Eager warmup for ALL capture sizes
for size in self.capture_sizes:
dummy = torch.zeros(size, hidden_dim, device=device, dtype=dtype)
with torch.no_grad():
self._forward_cudagraph_compatible(dummy)
torch.cuda.synchronize(device)
# Phase 2: Capture graphs
for size in self.capture_sizes:
try:
self._capture_graph_for_size(size, device, dtype, hidden_dim)
logger.info(" Captured CUDA Graph for batch_size=%d", size)
except Exception:
logger.warning(
" Failed to capture CUDA Graph for batch_size=%d",
size,
exc_info=True,
)
self.enabled = True
self._warmed_up = True
logger.info(
"CUDAGraphAcousticTransformerWrapper warmup complete. Captured %d/%d graphs.",
len(self.graphs),
len(self.capture_sizes),
)
def _forward_cudagraph_compatible(self, hidden_states: torch.Tensor, noise: torch.Tensor | None = None):
"""
The actual computation captured by the CUDA graph.
This replaces the full compute_mm_logits -> acoustic_transformer.forward() path
with a graph-compatible version:
- Uses argmax instead of vLLM Sampler (equivalent for top_k=1)
- Uses pre-created timesteps buffer instead of torch.linspace
- Uses pre-created scalar tensors for torch.where
- Calls _predict_velocity directly
- Uses a pre-allocated noise buffer to avoid baking random state
into the CUDA graph
"""
at = self.acoustic_transformer
B = hidden_states.shape[0]
# --- Semantic logits via linear projection ---
semantic_logit = at.semantic_codebook_output(hidden_states).float()
semantic_logit[:, self.empty_audio_token_id] = -float("inf")
semantic_logit[:, self.semantic_mask_start :] = -float("inf")
# argmax == top_k=1 greedy sampling
semantic_code = semantic_logit.argmax(dim=-1, keepdim=True) # (B, 1)
# --- Flow matching: Euler ODE ---
should_decode = semantic_code.squeeze(1) != self.end_audio_token_id
if noise is not None:
x = noise
else:
x = torch.randn(B, self.n_acoustic_codebook, device=hidden_states.device, dtype=hidden_states.dtype)
# Pre-compute zero hidden states for unconditional CFG branch
hidden_states_zero = torch.zeros_like(hidden_states)
timesteps = self.timesteps
for i in range(len(timesteps) - 1):
t = timesteps[i]
dt = timesteps[i + 1] - timesteps[i]
# Batch conditional + unconditional velocity in a single forward pass
t_emb = at.time_embedding(t.view(-1, 1).repeat(B, 1)).to(hidden_states.dtype)
x_batched = torch.cat([x, x], dim=0) # (2B, C)
llm_batched = torch.cat([hidden_states, hidden_states_zero], dim=0) # (2B, D)
t_emb_batched = t_emb.repeat(2, 1) # (2B, D)
v_all = at._predict_velocity(x_t=x_batched, llm_output=llm_batched, t_emb=t_emb_batched)
v_t, uncond_v_t = v_all[:B], v_all[B:]
# CFG combination
v_t = self.cfg_alpha * v_t + (1 - self.cfg_alpha) * uncond_v_t
x = x + v_t * dt
# --- Quantize ---
sampled = torch.clamp(x, -1, 1)
scaled_x = ((sampled + 1) / 2) * (self.acoustic_embeddings_levels - 1)
output_codes = scaled_x.round().long()
output_codes[~should_decode] = self.empty_audio_token_id
acoustic_codes = output_codes + len(AudioSpecialTokens)
# --- Combine semantic + acoustic ---
audio_codes = torch.cat([semantic_code, acoustic_codes], dim=1) # (B, 1 + n_acoustic)
# --- Compute fake_eos ---
fake_eos = torch.where(
audio_codes[:, 0] == self.end_audio_token_id,
self.fake_eos_one,
self.fake_eos_zero,
)
return fake_eos, audio_codes
def _capture_graph_for_size(
self,
size: int,
device: torch.device,
dtype: torch.dtype,
hidden_dim: int,
):
"""Capture a CUDA graph for a specific batch size."""
static_input = torch.zeros(size, hidden_dim, device=device, dtype=dtype)
static_noise = torch.randn(size, self.n_acoustic_codebook, device=device, dtype=dtype)
# Stabilizing eager run
with torch.no_grad():
_ = self._forward_cudagraph_compatible(static_input, noise=static_noise)
torch.cuda.synchronize(device)
graph = CUDAGraph()
with torch.no_grad():
with torch.cuda.graph(graph):
static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible(
static_input, noise=static_noise
)
self.graphs[size] = graph
self.static_inputs[size] = static_input
self.static_noise[size] = static_noise
self.static_fake_eos[size] = static_fake_eos
self.static_audio_codes[size] = static_audio_codes
def _get_padded_size(self, actual_size: int) -> int | None:
"""Round up to the nearest captured bucket size."""
for size in self.capture_sizes:
if actual_size <= size:
return size
return None
def __call__(
self,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, dict[str, list[torch.Tensor]] | None]:
"""
Drop-in replacement for model.compute_mm_logits().
Falls back to eager execution if:
- CUDA graph is not enabled/warmed up
- Batch size exceeds largest captured size
"""
actual_size = hidden_states.shape[0]
if not self.enabled or not self._warmed_up:
return self.model.compute_mm_logits(hidden_states)
padded_size = self._get_padded_size(actual_size)
if padded_size is None or padded_size not in self.graphs:
return self.model.compute_mm_logits(hidden_states)
# Zero static input, then copy actual data
self.static_inputs[padded_size].zero_()
self.static_inputs[padded_size][:actual_size] = hidden_states
# Fill noise buffer with fresh random values before replay so the
# flow-matching ODE starts from different initial noise each time.
self.static_noise[padded_size].normal_()
# Replay captured graph
self.graphs[padded_size].replay()
# Clone and slice outputs for actual batch size
fake_eos = self.static_fake_eos[padded_size][:actual_size].clone()
audio_codes = self.static_audio_codes[padded_size][:actual_size].clone()
# Package into expected format: (fake_eos, {"audio": [list of tensors]})
audio_list = list(torch.split(audio_codes.unsqueeze(1), 1, dim=0))
mm_tokens = {"audio": audio_list}
return fake_eos, mm_tokens
@@ -0,0 +1,372 @@
from collections.abc import Iterable
from functools import cached_property
from pathlib import Path
import regex as re
import torch
import torch.nn as nn
from huggingface_hub import hf_hub_download
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.voxtral_tts.cuda_graph_acoustic_transformer_wrapper import (
CUDAGraphAcousticTransformerWrapper,
)
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
VoxtralTTSDummyInputsBuilder,
VoxtralTTSMultiModalProcessor,
VoxtralTTSProcessingInfo,
)
logger = init_logger(__name__)
def parse_batched_audio_input(input_ids: torch.Tensor, num_codebooks: int) -> tuple[list[torch.Tensor], list[int]]:
"""Parse batched input_ids with [ctx_frames, context_length, ...tokens] format.
Each request in the batch is laid out as:
[ctx_frames, context_length, <flat audio tokens>]
where the flat audio tokens have length (ctx_frames + context_length) * num_codebooks.
Returns:
all_audio_tokens: list of (num_frames, num_codebooks) tensors per request.
all_ctx_frames: list of ctx_frames values per request.
"""
all_audio_tokens: list[torch.Tensor] = []
all_ctx_frames: list[int] = []
offset = 0
while offset < input_ids.numel():
ctx_frames = int(input_ids[offset].item())
context_length = int(input_ids[offset + 1].item())
offset += 2
req_input_ids_length = (ctx_frames + context_length) * num_codebooks
req_input_ids = input_ids[offset : offset + req_input_ids_length]
offset += req_input_ids_length
assert req_input_ids.numel() % num_codebooks == 0, (
f"Number of elements must be divisible by {num_codebooks} but get {req_input_ids.numel()=}"
)
num_of_audios = req_input_ids.numel() // num_codebooks
audio_tokens = req_input_ids.view(num_of_audios, num_codebooks)
all_audio_tokens.append(audio_tokens)
all_ctx_frames.append(ctx_frames)
return all_audio_tokens, all_ctx_frames
def apply_ctx_frames_cutting(
batch_audio_arrays: list[torch.Tensor],
all_ctx_frames: list[int],
downsample_factor: int,
) -> list[torch.Tensor]:
"""Cut leading context samples from decoded audio arrays.
Args:
batch_audio_arrays: raw decoded audio arrays, one per request.
all_ctx_frames: number of context frames per request.
downsample_factor: samples per frame in the audio tokenizer.
Returns:
List of audio arrays with context samples removed.
"""
result: list[torch.Tensor] = []
for audio_array, ctx_frames in zip(batch_audio_arrays, all_ctx_frames):
if ctx_frames > 0:
new_sample_cut = downsample_factor * ctx_frames
audio_array = audio_array[new_sample_cut:]
result.append(audio_array)
return result
@MULTIMODAL_REGISTRY.register_processor(
VoxtralTTSMultiModalProcessor,
info=VoxtralTTSProcessingInfo,
dummy_inputs=VoxtralTTSDummyInputsBuilder,
)
class VoxtralTTSForConditionalGeneration(
nn.Module,
SupportsMultiModal,
CustomProcessMixin,
):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.config = config
self.repo_id = vllm_config.model_config.model
self.is_hf_model = not Path(self.repo_id).is_dir()
self.model_stage = vllm_config.model_config.model_stage
if self.model_stage == "audio_generation":
self.has_preprocess = True
self.has_postprocess = True
self.requires_raw_input_tokens = True
self.set_custom_preprocess(self.tts_preprocess)
self.set_custom_postprocess(self.tts_postprocess)
self.audio_generation = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config,
prefix=maybe_prefix(prefix, "audio_generation"),
architectures=["VoxtralTTSAudioGeneration"],
)
self.model = self.audio_generation
self.audio_tokenizer = None
self._cudagraph_acoustic_transformer = None
self._vllm_config = vllm_config
tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
self._audio_token_id = tokenizer.instruct.audio_encoder.special_ids.audio
speaker_id = config.audio_config.get("speaker_id", None)
if speaker_id and self.is_hf_model:
self.voice_to_embedding = {
sid: torch.load(hf_hub_download(repo_id=self.repo_id, filename=f"voice_embedding/{sid}.pt"))
for sid in speaker_id
}
logger.info("Available voice embeddings: %s", list(self.voice_to_embedding.keys()))
else:
self.voice_to_embedding = {}
logger.warning("No speaker_id configured in audio_config.No voice embeddings will be available.")
elif self.model_stage == "audio_tokenizer":
self.requires_raw_input_tokens = True
self.audio_generation = None
self.audio_tokenizer = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config,
prefix=maybe_prefix(prefix, "audio_tokenizer"),
architectures=["VoxtralTTSAudioTokenizer"],
)
self.model = self.audio_tokenizer
else:
raise ValueError("Invalid model stage")
# -------------------- CUDA Graph for acoustic transformer --------------------
def _enable_acoustic_transformer_cudagraph(self):
"""Initialize and capture CUDA graphs for compute_mm_logits."""
if self.model_stage != "audio_generation" or not hasattr(self, "_vllm_config"):
return
model_cfg = getattr(self._vllm_config, "model_config", None)
if model_cfg and getattr(model_cfg, "enforce_eager", False):
logger.info("CUDA Graph for acoustic transformer not enabled: --enforce-eager is set")
return
try:
acoustic_transformer = self.model.acoustic_transformer
device = next(acoustic_transformer.parameters()).device
dtype = next(acoustic_transformer.parameters()).dtype
hidden_dim = self.model.config.text_config.hidden_size
wrapper = CUDAGraphAcousticTransformerWrapper(self.model)
wrapper._warmup_and_capture(device, dtype, hidden_dim)
self._cudagraph_acoustic_transformer = wrapper
logger.info("CUDA Graph for acoustic transformer enabled")
except Exception:
logger.warning("Failed to enable CUDA Graph for acoustic transformer", exc_info=True)
self._cudagraph_acoustic_transformer = None
# -------------------- Device utilities --------------------
@cached_property
def sampler(self):
if hasattr(self.model, "sampler"):
return self.model.sampler
return Sampler()
def tts_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict | None):
self.post_process_idx = 0
audio_tokens = info_dict.pop("audio", None)
if audio_tokens is not None:
kwargs = {"audio_tokens": audio_tokens.to(input_ids.device)}
multimodal_embeddings = self.model.embed_multimodal(**kwargs)
if input_ids[0] == self._audio_token_id:
input_embeds = multimodal_embeddings[0]
return input_ids, input_embeds, info_dict
voice = info_dict.pop("voice", None)
if voice is not None:
if isinstance(voice, list):
voice = voice[0]
multimodal_embeddings = self.voice_to_embedding[voice].to(input_ids.device).clone().detach()
is_multimodal = input_ids == self._audio_token_id
input_embeds = self.embed_input_ids(
input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal
)
return input_ids, input_embeds, info_dict
return input_ids, input_embeds, info_dict
def tts_postprocess(self, hidden_states: torch.Tensor, multimodal_outputs: object, **info_dict: object | None):
update_dict = {}
if isinstance(multimodal_outputs, dict) and "audio" in multimodal_outputs:
assert self.post_process_idx < len(multimodal_outputs["audio"]), (
f"Expect {self.post_process_idx=} < {len(multimodal_outputs['audio'])=}"
)
update_dict["audio"] = multimodal_outputs["audio"][self.post_process_idx]
self.post_process_idx += 1
return update_dict
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings=None,
is_multimodal=None,
) -> torch.Tensor:
if self.model_stage == "audio_tokenizer":
return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.config.text_config.hidden_size)
return self.model.embed_input_ids(
input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal
)
def embed_multimodal(self, **kwargs):
# Delegate to generation model for multimodal processing
return self.model.embed_multimodal(**kwargs)
def last_index_of(self, list, value):
return len(list) - 1 - list[::-1].index(value)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors | OmniOutput:
if self.model_stage == "audio_generation":
if inputs_embeds is not None:
input_ids = None
hidden_states = self.model(
input_ids,
positions,
intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return hidden_states
if self.model_stage == "audio_tokenizer":
if (input_ids == 0).all():
logger.info("audio_tokenizer: sample run with dummy input")
# TODO(chenyo): Move this to dummy_inputs creation
num_codebooks = self.audio_tokenizer.num_codebooks
audio_tokens = torch.randint(low=2, high=100, size=(116, num_codebooks), dtype=torch.int32)
audio_tokens[-1, :] = 0
audio_tokens[-1, 0] = 1
batch_audio_arrays = self.audio_tokenizer.decode_helper_batch_async([audio_tokens])
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"audio": batch_audio_arrays},
)
all_audio_tokens, all_ctx_frames = parse_batched_audio_input(
input_ids, num_codebooks=self.audio_tokenizer.num_codebooks
)
# Batch decode all requests at once
batch_audio_arrays_raw = self.audio_tokenizer.decode_helper_batch_async(all_audio_tokens)
batch_audio_arrays = apply_ctx_frames_cutting(
batch_audio_arrays_raw, all_ctx_frames, self.audio_tokenizer.downsample_factor
)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"audio": batch_audio_arrays},
)
def make_omni_output(
self, model_outputs: torch.Tensor | OmniOutput | tuple, logits_index: int | None = None, **kwargs
) -> OmniOutput:
if isinstance(model_outputs, torch.Tensor):
if self.model_stage == "audio_generation":
hidden_states = model_outputs
assert logits_index is not None
input_hidden_states = hidden_states[logits_index]
if self._cudagraph_acoustic_transformer is not None:
fake_eos, multimodal_outputs = self._cudagraph_acoustic_transformer(input_hidden_states)
else:
fake_eos, multimodal_outputs = self.model.compute_mm_logits(input_hidden_states)
hidden_states[logits_index, 0] = fake_eos
return OmniOutput(
text_hidden_states=hidden_states,
multimodal_outputs=multimodal_outputs,
)
raise ValueError(f"Unsupported {self.model_stage} for model_outputs type: {type(model_outputs)}")
if isinstance(model_outputs, OmniOutput):
if self.model_stage == "audio_tokenizer":
return model_outputs
raise ValueError(f"Unsupported {self.model_stage} for model_outputs type: {type(model_outputs)}")
raise ValueError(f"Unsupported model_outputs type: {type(model_outputs)}")
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
# Handle OmniOutput type
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if self.model_stage == "audio_generation":
text_logits = self.model.fake_logits_for_audio_tokens(fake_eos=hidden_states)
return text_logits
if self.model_stage == "audio_tokenizer":
raise ValueError("Invalid model stage")
def sample(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> SamplerOutput | None:
return self.model.sample(logits, sampling_metadata)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loaded_weights = set()
remapping_rules = [
(r"^audio_tokenizer\.(.*)$", r"\1"), # noqa: E501
(
r"^mm_audio_embeddings\.audio_codebook_embeddings\.embeddings\.(weight|bias)",
r"audio_token_embedding.embeddings.\1",
), # noqa: E501
(r"^mm_audio_embeddings\.tok_embeddings\.weight", r"tok_embeddings.weight"), # noqa: E501
]
def llm_weights_generator():
nonlocal loaded_weights
for name, w in weights:
is_audio_tokenizer = name.startswith(
"mm_audio_embeddings.audio_codebook_embeddings"
) or name.startswith("audio_tokenizer.")
if is_audio_tokenizer and self.audio_tokenizer is not None:
# Remap name only when loading audio_tokenizer (Stage-1).
# Yield the original name in Stage-0 so audio_generation.load_weights()
# can still filter by prefix using its own remapping rules.
remapped = name
for pattern, repl in remapping_rules:
if re.fullmatch(pattern, remapped):
remapped = re.sub(pattern, repl, remapped)
name = self.audio_tokenizer.load_weight((remapped, w))
loaded_weights.add(f"audio_tokenizer.{name}")
continue
yield (name, w)
audio_generation_weights = list(llm_weights_generator())
if self.audio_generation is not None:
for name in self.audio_generation.load_weights(audio_generation_weights):
loaded_weights.add(f"audio_generation.{name}")
# If encoder weights were not in the checkpoint, mark them as
# "loaded" so the weight-validation does not fail.
# encode_waveforms() will raise at runtime if called without encoder weights.
if self.audio_tokenizer is not None and not self.audio_tokenizer._encoder_loaded:
for name, _ in self.audio_tokenizer.named_parameters():
if name.startswith(self.audio_tokenizer._encoder_weight_prefixes):
loaded_weights.add(f"audio_tokenizer.{name}")
# Capture CUDA graphs for compute_mm_logits after weights are loaded
self._enable_acoustic_transformer_cudagraph()
return loaded_weights
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff