[New Model] support nemotron Audex (#4976)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-07-28 03:58:22 +00:00
1 parent f2c5152df0
commit 5014fea4a0
64 files changed
+7760 -39

No files matched your search

@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,227 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Audex code2wav: streaming speech-codec-token → waveform stage.
Wraps the vendored Audex causal speech decoder (see ``speech_decoder/``).
Unlike the flow/vocoder stages of other TTS models, the decoder is natively
stateful and streaming: one ``AudexCausalSpeechDecoderSession`` per request
holds the decode cache, each engine step pushes only the newly received codec
frames, and the session's 4-frame lookahead tail is drained by ``flush()``
when the request finishes. The stage therefore never re-decodes left context
and always returns delta audio.
"""
from collections.abc import Iterable
from typing import Any
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm_omni.model_executor.models.audex.speech_decoder import (
AudexCausalSpeechDecoderConfig,
AudexCausalSpeechDecoderModel,
AudexCausalSpeechDecoderSession,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
# Frames per decoder emit. 5 frames = 100 ms at 50 fps — the granularity the
# official streaming demo uses; small enough for a fast first audible chunk.
_DEFAULT_EMIT_CHUNK_FRAMES = 5
def _meta_int(value: Any, default: int = 0) -> int:
if isinstance(value, torch.Tensor):
return int(value.reshape(-1)[0].item()) if value.numel() else default
if isinstance(value, list | tuple):
return _meta_int(value[0], default) if value else default
try:
return int(value)
except (TypeError, ValueError):
return default
def _meta_bool(value: Any) -> bool:
if isinstance(value, torch.Tensor):
return bool(value.reshape(-1)[0].item()) if value.numel() else False
if isinstance(value, list | tuple):
return _meta_bool(value[0]) if value else False
return bool(value)
def _meta_str(value: Any) -> str | None:
if isinstance(value, list | tuple):
return _meta_str(value[0]) if value else None
if isinstance(value, str):
return value
return None
def _codes_from_runtime_info(runtime_info: dict[str, Any] | None) -> list[int]:
"""Extract this step's new codec frame ids from the connector payload."""
if not isinstance(runtime_info, dict):
return []
codes = runtime_info.get("codes")
if not isinstance(codes, dict):
return []
audio = codes.get("audio")
if isinstance(audio, torch.Tensor):
return audio.reshape(-1).tolist()
if isinstance(audio, list | tuple):
return [int(c) for c in audio]
return []
class AudexCode2Wav(nn.Module):
"""Stage-1 model for Audex TTS (GenerationModelRunner)."""
input_modalities = "audio"
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
# Build the decoder skeleton from config so vLLM's memory profiler
# sees it at startup; weights load later in load_weights().
config = AudexCausalSpeechDecoderConfig.from_pretrained(self.model_path)
self.decoder = AudexCausalSpeechDecoderModel(config)
self.decoder.eval()
self._sample_rate = int(config.sample_rate)
extra_cfg = self._connector_extra_config()
self._emit_chunk_frames = _meta_int(extra_cfg.get("decoder_emit_chunk_frames"), _DEFAULT_EMIT_CHUNK_FRAMES)
if self._emit_chunk_frames <= 0:
raise ValueError(f"decoder_emit_chunk_frames must be positive, got {self._emit_chunk_frames}")
# One streaming session per in-flight request, keyed by the request id
# carried in each chunk payload's meta (equal to the stage-1 engine
# request id, so on_requests_finished can free aborted requests too).
self._sessions: dict[str, AudexCausalSpeechDecoderSession] = {}
def _connector_extra_config(self) -> dict[str, Any]:
connector_cfg = getattr(self.vllm_config.model_config, "stage_connector_config", None)
if isinstance(connector_cfg, dict):
extra = connector_cfg.get("extra", connector_cfg)
return extra if isinstance(extra, dict) else {}
extra = getattr(connector_cfg, "extra", None)
return extra if isinstance(extra, dict) else {}
# -------------------- vLLM model interface --------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
# This stage ignores token embeddings; keep a stable dummy for the runner.
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
return None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
state_dict = dict(weights)
self.decoder.load_state_dict(state_dict, strict=True)
device = self.vllm_config.device_config.device
self.decoder.to(device=device, dtype=torch.float32)
return {f"decoder.{name}" for name in state_dict}
def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None:
"""Free decoder sessions for finished/aborted requests."""
for req_id in finished_req_ids:
self._sessions.pop(req_id, None)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
# The runner may unpack the NamedTuple in transit; rebuild it.
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
return OmniOutput(*model_outputs)
raise TypeError(f"AudexCode2Wav expected OmniOutput, got {type(model_outputs)}")
# -------------------- decoding --------------------
def _session_for(self, req_id: str) -> AudexCausalSpeechDecoderSession:
session = self._sessions.get(req_id)
if session is None:
session = self.decoder.create_session(
chunk_frames=self._emit_chunk_frames,
return_numpy=False,
)
self._sessions[req_id] = session
return session
def _decode_request(self, req_id: str, codes: list[int], finished: bool) -> torch.Tensor:
"""Push new frames into the request's session; return this step's delta audio."""
if finished and not codes and req_id not in self._sessions:
# Zero codec tokens for the whole request. Do NOT raise: an
# exception in forward() kills the engine core for every request.
# Returning empty audio lets the serving layer fail this request.
logger.error("Audex code2wav: request %s finished with no codec tokens; no audio generated", req_id)
return torch.empty(0, dtype=torch.float32)
chunks: list[torch.Tensor] = []
session = self._session_for(req_id)
with torch.inference_mode():
if codes:
for _sr, chunk in session.push([[int(c)] for c in codes]):
chunks.append(chunk.reshape(-1))
if finished:
for _sr, chunk in session.flush():
chunks.append(chunk.reshape(-1))
if finished:
self._sessions.pop(req_id, None)
if not chunks:
return torch.empty(0, dtype=torch.float32)
return torch.cat(chunks).to(torch.float32)
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
runtime_infos = runtime_additional_information or []
# One output entry per scheduled request, even on steps where a
# request's chunk payload has not arrived yet.
seq_token_counts = kwargs.get("seq_token_counts")
if seq_token_counts is not None:
num_reqs = len(seq_token_counts)
elif input_ids is not None and input_ids.numel() > 0 and not runtime_infos:
num_reqs = 1
else:
num_reqs = len(runtime_infos)
audios: list[torch.Tensor] = []
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
for i in range(num_reqs):
info = runtime_infos[i] if i < len(runtime_infos) else {}
info = info if isinstance(info, dict) else {}
meta = info.get("meta", {})
meta = meta if isinstance(meta, dict) else {}
req_id = _meta_str(meta.get("req_id"))
# async_chunk delivers ``stream_finished`` (the adapter strips
# ``finished``); the sync full-payload path delivers ``finished``.
finished = _meta_bool(meta.get("stream_finished")) or _meta_bool(meta.get("finished"))
codes = _codes_from_runtime_info(info)
if req_id is None:
if codes:
raise ValueError("Audex code2wav chunk payload is missing meta.req_id")
audios.append(torch.empty(0, dtype=torch.float32))
continue
audios.append(self._decode_request(req_id, codes, finished))
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": [sr_tensor] * num_reqs},
)
@@ -0,0 +1,322 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/modeling_audex_vllm.py
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/audio_encoder.py
"""Audex audio-understanding model (``checkpoint_folder_full``): S2T / audio QA.
Mirrors the Qwen2-Audio pattern with the 2B dense backbone:
* ``language_model`` = the vllm-omni ``NemotronDenseForCausalLM`` port
(wrapped via ``init_vllm_registered_model``; resolved through the omni
model registry).
* ``audio_tower`` = transformers ``Qwen2AudioEncoder`` ("NV-Whisper");
a parameter-free avg pool maps 1500 → 750 frames per padded 30 s clip.
* ``audio_projector`` = RMSNorm → fc1 → relu² → fc2 (no bias), matching the
release ``NemotronHAudexProjector``.
``embed_multimodal(...)`` runs encoder+projector and returns one embedding
tensor per audio item; vLLM's ``SupportsMultiModal.embed_input_ids`` merges
them into the ``<so_embedding>`` placeholder positions. The encoder runs in
microbatches over clips so peak memory is bounded regardless of audio length
(up to the 900 s / 30-clip cap enforced by the processor).
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import Annotated, Literal
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.model_executor.models.interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
SupportsMambaPrefixCaching,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.utils import (
init_vllm_registered_model,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm_omni.model_executor.models.audex.audex_omni_processing import (
AUDIO_ENCODER_MICRO_BATCH,
SOUND_TOKEN,
AudexDummyInputsBuilder,
AudexMultiModalProcessor,
AudexProcessingInfo,
)
class AudexProjector(nn.Module):
"""Megatron sound_projection equivalent (matches HF NemotronHAudexProjector)."""
def __init__(
self,
audio_encoder_hidden_size: int,
intermediate_size: int,
text_hidden_size: int,
activation: str = "relu2",
norm_eps: float = 1e-5,
) -> None:
super().__init__()
self.norm_weight = nn.Parameter(torch.ones(audio_encoder_hidden_size))
self.norm_eps = norm_eps
self.fc1 = nn.Linear(audio_encoder_hidden_size, intermediate_size, bias=False)
self.fc2 = nn.Linear(intermediate_size, text_hidden_size, bias=False)
self.activation = activation
def _rms_norm(self, x: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x = x.float()
variance = x.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + self.norm_eps)
return (self.norm_weight.float() * x).to(dtype)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
hidden_states = self._rms_norm(hidden_states)
hidden_states = self.fc1(hidden_states)
if self.activation == "relu2":
hidden_states = F.relu(hidden_states).pow(2)
elif self.activation == "gelu":
hidden_states = F.gelu(hidden_states)
else:
raise ValueError(f"Unsupported audio projector activation: {self.activation}")
return self.fc2(hidden_states)
def _build_qwen2_audio_encoder(audio_config: dict) -> nn.Module:
from transformers.models.qwen2_audio.configuration_qwen2_audio import (
Qwen2AudioEncoderConfig,
)
from transformers.models.qwen2_audio.modeling_qwen2_audio import Qwen2AudioEncoder
cfg = Qwen2AudioEncoderConfig(**dict(audio_config))
return Qwen2AudioEncoder(cfg)
class AudexAudioFeatureInputs(TensorSchema):
"""
Dimensions:
- c: total number of 30s clips (flattened across audio items)
- b: number of audio items
"""
type: Literal["audio_features"] = "audio_features"
input_features: Annotated[torch.Tensor, TensorShape("c", 128, 3000)]
audio_num_clips: Annotated[list[int], None] = None
@MULTIMODAL_REGISTRY.register_processor(
AudexMultiModalProcessor,
info=AudexProcessingInfo,
dummy_inputs=AudexDummyInputsBuilder,
)
class NemotronDenseAudexForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsPP,
):
_LM_ARCHITECTURE = "NemotronDenseForCausalLM"
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("audio"):
return SOUND_TOKEN
raise ValueError(f"Only audio modality is supported, got {modality}")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config,
prefix=maybe_prefix(prefix, "language_model"),
architectures=[self._LM_ARCHITECTURE],
)
llm_dtype = self.language_model.config.dtype
assert isinstance(llm_dtype, torch.dtype)
self.llm_dtype = llm_dtype
with self._mark_tower_model(vllm_config, "audio"):
self.audio_tower = _build_qwen2_audio_encoder(config.audio_config).to(llm_dtype)
self.audio_projector = AudexProjector(
audio_encoder_hidden_size=config.audio_encoder_hidden_size,
intermediate_size=config.audio_projector_intermediate_size,
text_hidden_size=config.hidden_size,
activation=config.audio_projector_activation,
norm_eps=config.audio_projector_norm_eps,
).to(llm_dtype)
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
def get_language_model(self) -> nn.Module:
return self.language_model
def _parse_and_validate_audio_input(self, **kwargs: object):
input_features = kwargs.pop("input_features", None)
audio_num_clips = kwargs.pop("audio_num_clips", None)
if input_features is None:
return None
return AudexAudioFeatureInputs(
type="audio_features",
input_features=input_features,
audio_num_clips=audio_num_clips,
validate=False,
)
@torch.inference_mode()
def _encode_clips(self, input_features: torch.Tensor) -> torch.Tensor:
"""(C, 128, 3000) -> (C, 750, hidden) via microbatched encoder+projector."""
param = next(self.audio_tower.parameters())
input_features = input_features.to(device=param.device, dtype=param.dtype)
out_chunks = []
for start in range(0, input_features.shape[0], AUDIO_ENCODER_MICRO_BATCH):
chunk = input_features[start : start + AUDIO_ENCODER_MICRO_BATCH]
hidden = self.audio_tower(input_features=chunk, return_dict=True).last_hidden_state
out_chunks.append(self.audio_projector(hidden))
return torch.cat(out_chunks, dim=0)
def _process_audio_input(self, audio_input) -> tuple[torch.Tensor, ...]:
input_features = audio_input["input_features"]
audio_num_clips = audio_input["audio_num_clips"]
if input_features.ndim == 4:
input_features = input_features.flatten(0, 1)
projected = self._encode_clips(input_features)
per_clip = projected.reshape(projected.shape[0], -1, projected.shape[-1])
if audio_num_clips is None:
num_clips_list = [projected.shape[0]]
elif isinstance(audio_num_clips, torch.Tensor):
num_clips_list = audio_num_clips.flatten().tolist()
else:
num_clips_list = [int(x) for x in audio_num_clips]
grouped: list[torch.Tensor] = []
offset = 0
for n in num_clips_list:
clips = per_clip[offset : offset + n]
grouped.append(clips.reshape(-1, clips.shape[-1]))
offset += n
return tuple(grouped)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
audio_input = self._parse_and_validate_audio_input(**kwargs)
if audio_input is None:
return []
return self._process_audio_input(audio_input)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
return self.language_model.model(input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds)
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata=None) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Split the combined weight stream into LLM / audio_tower / projector.
vLLM streams *all* ``.safetensors`` in the model dir; route
``audio_encoder.*``/``audio_projector.*`` to the audio modules and
leave the remaining ``model.*``/``lm_head.*`` names untouched so the
wrapped NemotronDenseForCausalLM loader applies its own qkv fusion
and remapping.
"""
loaded: set[str] = set()
lm_weights: list[tuple[str, torch.Tensor]] = []
enc_sd: dict[str, torch.Tensor] = {}
proj_map: dict[str, torch.Tensor] = {}
for name, w in weights:
if name.startswith("audio_encoder."):
enc_sd[name[len("audio_encoder.") :]] = w
elif name == "audio_projector.norm.weight":
proj_map["norm_weight"] = w
elif name.startswith("audio_projector."):
proj_map[name[len("audio_projector.") :]] = w
else:
lm_weights.append((name, w))
lm_loaded = self.language_model.load_weights(lm_weights)
loaded |= {f"language_model.{n}" for n in lm_loaded}
if not enc_sd:
raise ValueError("No audio_encoder.* weights found in checkpoint stream.")
missing, unexpected = self.audio_tower.load_state_dict(enc_sd, strict=False)
if missing:
raise ValueError(f"Missing audio_tower params: {list(missing)[:8]}")
if unexpected:
raise ValueError(f"Unexpected audio_tower keys: {list(unexpected)[:8]}")
loaded |= {f"audio_tower.{k}" for k in enc_sd}
if set(proj_map) != {"norm_weight", "fc1.weight", "fc2.weight"}:
raise ValueError(f"Unexpected audio_projector keys: {sorted(proj_map)}")
self.audio_projector.load_state_dict(proj_map, strict=True)
loaded |= {f"audio_projector.{k}" for k in proj_map}
return loaded
@MULTIMODAL_REGISTRY.register_processor(
AudexMultiModalProcessor,
info=AudexProcessingInfo,
dummy_inputs=AudexDummyInputsBuilder,
)
class NemotronHAudexForConditionalGeneration(
NemotronDenseAudexForConditionalGeneration,
IsHybrid,
HasInnerState,
SupportsMambaPrefixCaching,
):
"""Audex 30B-A3B audio-understanding model (``checkpoint_folder_full``).
Mirrors nvidia/Nemotron-Labs-Audex-30B-A3B's
``inference_scripts_vllm/audioqa_scripts/audex_30b_a3b_vllm/modeling_audex_vllm.py``:
the same encoder/projector/processing as the dense 2B wrapper around
vLLM's native hybrid Mamba+MoE ``NemotronHForCausalLM``, whose state
hooks are delegated below.
"""
_LM_ARCHITECTURE = "NemotronHForCausalLM"
is_hybrid = True
has_inner_state = True
supports_mamba_prefix_caching = True
@classmethod
def get_mamba_state_shape_from_config(cls, vllm_config):
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
return NemotronHForCausalLM.get_mamba_state_shape_from_config(vllm_config)
@classmethod
def get_mamba_state_dtype_from_config(cls, vllm_config):
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
return NemotronHForCausalLM.get_mamba_state_dtype_from_config(vllm_config)
@classmethod
def get_mamba_state_copy_func(cls):
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
return NemotronHForCausalLM.get_mamba_state_copy_func()
@@ -0,0 +1,260 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/audio_features.py
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/processing_audex_vllm.py
"""Multimodal processing for Audex audio understanding (S2T / audio QA).
No unified HF ``ProcessorMixin`` exists for Audex (only a
``WhisperFeatureExtractor`` under ``audio_preprocessor/``), so
``_call_hf_processor`` is fully overridden: it tokenizes the prompt and
builds NV-Whisper features with 30 s non-overlapping windows (last clip
zero-padded), exactly mirroring the release HF path.
Placeholder expansion uses vLLM's ``PromptReplacement``: a single
``<so_embedding>`` per audio expands to ``<so_start>`` + N*``<so_embedding>``
+ ``<so_end>`` where N = num_clips * 750.
"""
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
import numpy as np
import torch
from transformers import BatchFeature
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
SOUND_TOKEN = "<so_embedding>"
SOUND_START_TOKEN = "<so_start>"
SOUND_END_TOKEN = "<so_end>"
SOUND_EMBEDDING_SIZE = 750
SOUND_CLIP_DURATION = 30.0
SOUND_TARGET_RATE = 16000
MAX_AUDIO_SECONDS = 900
MAX_AUDIO_CLIPS = 30
MAX_AUDIO_TOKENS = MAX_AUDIO_CLIPS * SOUND_EMBEDDING_SIZE
AUDIO_ENCODER_MICRO_BATCH = 8
# ---------------------------------------------------------------- features
def normalize_audio(audio: np.ndarray) -> np.ndarray:
"""Return mono float32 audio in [-1, 1], matching the Megatron/HF path."""
audio = np.asarray(audio)
if audio.ndim == 2:
if audio.shape[1] <= 2:
audio = audio.mean(axis=1)
elif audio.shape[0] <= 2:
audio = audio.mean(axis=0)
else:
raise ValueError(f"Unsupported audio shape: {audio.shape}")
if audio.dtype == np.int16:
audio = audio.astype(np.float32) / 32768.0
elif audio.dtype != np.float32:
audio = audio.astype(np.float32)
max_abs = float(np.abs(audio).max()) if audio.size else 0.0
if max_abs > 1.0:
audio = audio / max_abs
return audio.astype(np.float32, copy=False)
def split_audio_into_clips(
audio: np.ndarray,
sample_rate: int = SOUND_TARGET_RATE,
clip_duration: float = SOUND_CLIP_DURATION,
) -> list[np.ndarray]:
"""Split audio into fixed clips; pad the final clip to full length."""
audio = normalize_audio(audio)
clip_samples = int(round(sample_rate * clip_duration))
if clip_samples <= 0:
raise ValueError(f"Invalid clip_samples: {clip_samples}")
if audio.size == 0:
audio = np.zeros(1, dtype=np.float32)
num_clips = max(1, math.ceil(audio.shape[0] / clip_samples))
clips: list[np.ndarray] = []
for idx in range(num_clips):
start = idx * clip_samples
clip = audio[start : start + clip_samples]
if clip.shape[0] < clip_samples:
clip = np.pad(clip, (0, clip_samples - clip.shape[0]))
clips.append(clip.astype(np.float32, copy=False))
return clips
def extract_whisper_features(
feature_extractor,
audio: np.ndarray,
sample_rate: int = SOUND_TARGET_RATE,
clip_duration: float = SOUND_CLIP_DURATION,
) -> torch.Tensor:
"""Return NV-Whisper input features shaped (num_clips, 128, 3000)."""
clips = split_audio_into_clips(audio, sample_rate=sample_rate, clip_duration=clip_duration)
features = feature_extractor(
clips,
sampling_rate=sample_rate,
return_tensors="pt",
padding="max_length",
return_attention_mask=False,
)
input_features = features.input_features
if input_features.ndim != 3:
raise ValueError(f"Expected 3D Whisper features, got {tuple(input_features.shape)}")
return input_features
# ---------------------------------------------------------------- processor
class AudexProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.model_config.hf_config
def get_feature_extractor(self, **kwargs: object):
from transformers import AutoFeatureExtractor
config = self.get_hf_config()
path = getattr(config, "audio_preprocessor_path", None) or "audio_preprocessor"
candidate = Path(path)
if not candidate.is_absolute():
candidate = Path(self.ctx.model_config.model) / candidate
return AutoFeatureExtractor.from_pretrained(str(candidate))
def get_data_parser(self):
return MultiModalDataParser(target_sr=SOUND_TARGET_RATE, target_channels=1)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": 1}
def get_mm_max_tokens_per_item(self, seq_len: int, mm_counts: Mapping[str, int] | None = None) -> Mapping[str, int]:
return {"audio": MAX_AUDIO_TOKENS}
def sound_token_ids(self) -> tuple[int, int, int]:
cfg = self.get_hf_config()
return (
int(cfg.sound_start_token_id),
int(cfg.sound_token_id),
int(cfg.sound_end_token_id),
)
class AudexDummyInputsBuilder(BaseDummyInputsBuilder[AudexProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return SOUND_TOKEN * mm_counts.get("audio", 0)
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_len = int(SOUND_CLIP_DURATION * SOUND_TARGET_RATE)
overrides = mm_options.get("audio") if mm_options else None
return {"audio": self._get_dummy_audios(length=audio_len, num_audios=num_audios, overrides=overrides)}
class AudexMultiModalProcessor(BaseMultiModalProcessor[AudexProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, Any],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
prompt_ids = tokenizer(prompt, add_special_tokens=False).input_ids
audios = mm_data.get("audios", []) or mm_data.get("audio", [])
if not audios:
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
feature_extractor = self.info.get_feature_extractor()
all_features: list[torch.Tensor] = []
audio_num_clips: list[int] = []
for item in audios:
audio = item[0] if isinstance(item, tuple) else item
audio = normalize_audio(np.asarray(audio))
duration = len(audio) / SOUND_TARGET_RATE
if duration > MAX_AUDIO_SECONDS:
raise ValueError(f"Audio duration {duration:.1f}s exceeds MAX_AUDIO_SECONDS={MAX_AUDIO_SECONDS}s.")
clips = split_audio_into_clips(audio, sample_rate=SOUND_TARGET_RATE, clip_duration=SOUND_CLIP_DURATION)
if len(clips) > MAX_AUDIO_CLIPS:
raise ValueError(f"Audio needs {len(clips)} clips > MAX_AUDIO_CLIPS={MAX_AUDIO_CLIPS}.")
feats = extract_whisper_features(
feature_extractor,
audio,
sample_rate=SOUND_TARGET_RATE,
clip_duration=SOUND_CLIP_DURATION,
)
all_features.append(feats)
audio_num_clips.append(feats.shape[0])
input_features = torch.cat(all_features, dim=0)
return BatchFeature(
dict(
input_ids=[prompt_ids],
input_features=input_features,
audio_num_clips=torch.tensor(audio_num_clips, dtype=torch.long),
),
tensor_type="pt",
)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
audio_num_clips = hf_inputs.get("audio_num_clips", torch.zeros(0, dtype=torch.long))
return dict(
input_features=MultiModalFieldConfig.flat_from_sizes("audio", audio_num_clips),
audio_num_clips=MultiModalFieldConfig.batched("audio"),
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
so_start, so_embed, so_end = self.info.sound_token_ids()
out_mm_data = out_mm_kwargs.get_data()
num_clips = out_mm_data.get("audio_num_clips", torch.zeros(0, dtype=torch.long))
num_clips_list = num_clips.flatten().tolist() if hasattr(num_clips, "flatten") else list(num_clips)
def get_replacement_audex(item_idx: int):
n = int(num_clips_list[item_idx]) * SOUND_EMBEDDING_SIZE
if n < 1:
raise ValueError(f"Audio item {item_idx} produced 0 embeddings.")
full = [so_start] + [so_embed] * n + [so_end]
return PromptUpdateDetails.select_token_id(full, so_embed)
return [
PromptReplacement(
modality="audio",
target=[so_embed],
replacement=get_replacement_audex,
)
]
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from the NemotronDense vLLM plugin bundled with
# nvidia/Nemotron-Labs-Audex-2B (nemotron_dense_vllm_plugin), which is not
# installed as a package; the model code is carried here instead.
"""Audex thinker: the Nemotron dense decoder-only LM that autoregressively
emits ``<speechcodec_N>`` tokens for TTS.
The thinker is a plain token-in/token-out vLLM model. Speech-codec token
extraction happens in the stage input processors
(``vllm_omni/model_executor/stage_input_processors/audex.py``), which read the
sampled token stream directly, so no omni-specific output plumbing is needed
here.
"""
from collections.abc import Iterable
from itertools import islice
from typing import Any
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
class NemotronDenseRMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-5):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
return self.weight * hidden_states.to(input_dtype)
class NemotronDenseMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
super().__init__()
self.up_proj = ColumnParallelLinear(
input_size=hidden_size,
output_size=intermediate_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.up_proj",
)
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
self.act_fn = get_act_fn(hidden_act)
def forward(self, x: torch.Tensor) -> torch.Tensor:
up, _ = self.up_proj(x)
x = self.act_fn(up)
x, _ = self.down_proj(x)
return x
class NemotronDenseAttention(nn.Module):
def __init__(
self,
config: Any,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
max_position_embeddings: int = 8192,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = getattr(config, "head_dim", None)
if self.head_dim is None:
self.head_dim = self.hidden_size // self.total_num_heads
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.max_position_embeddings = max_position_embeddings
self.qkv_proj = QKVParallelLinear(
hidden_size=hidden_size,
head_size=self.head_dim,
total_num_heads=self.total_num_heads,
total_num_kv_heads=self.total_num_kv_heads,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
input_size=self.total_num_heads * self.head_dim,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class NemotronDenseDecoderLayer(nn.Module):
def __init__(
self,
config: Any,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
attention_bias = getattr(config, "attention_bias", False) or getattr(config, "bias", False)
self.self_attn = NemotronDenseAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(config, "num_key_value_heads", config.num_attention_heads),
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
)
self.mlp = NemotronDenseMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=f"{prefix}.mlp",
)
self.input_layernorm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
self.post_attention_layernorm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
@support_torch_compile
class NemotronDenseModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
config.hidden_size,
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: NemotronDenseDecoderLayer(
config=config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(positions, hidden_states)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
hidden_states = self.norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
param = params_dict[scale_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
weight_loader(param, loaded_weight)
loaded_params.add(scale_name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class NemotronDenseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
"k_proj",
"v_proj",
],
}
embedding_modules = {
"embed_tokens": "input_embeddings",
"lm_head": "output_embeddings",
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
assert hasattr(config, "norm_eps") and hasattr(config, "num_hidden_layers")
self.config = config
self.quant_config = quant_config
self.model = NemotronDenseModel(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
self.lm_head.weight = self.model.embed_tokens.weight
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(config.vocab_size, scale=logit_scale)
else:
self.lm_head = PPMissingLayer()
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: Any,
) -> torch.Tensor | IntermediateTensors:
# The omni AR runner forwards extra bookkeeping kwargs (e.g.
# sampling_metadata); this plain LM ignores them.
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor,
sampling_metadata: Any = None,
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Audex TTA stage 1: XCodec1 RVQ codes → waveform.
Unlike the streaming causal speech decoder used for TTS, XCodec1 is a CNN
codec that the official flow decodes over the full sequence, so this stage
runs sync full-payload only: each request delivers its complete
de-interleaved ``[frames, 4]`` codec-id payload once at finish and the
decode happens in a single call.
The checkpoint (``hf-audio/xcodec-hubert-general-balanced``) is external to
the Audex repo and loads through transformers remote code; the model
skeleton is built from config at startup for vLLM's memory profiler and the
weights arrive via the standard ``load_weights`` iteration over the
snapshot's safetensors.
"""
from collections.abc import Iterable
from typing import Any
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm_omni.model_executor.models.audex.tta import (
XCODEC1_CODEBOOK_SIZE,
XCODEC1_NUM_CODEBOOKS,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
# Official cap on decoded frames (~10 s of audio at the codec frame rate).
_DEFAULT_MAX_TTA_FRAMES = 500
def _meta_str(value: Any) -> str | None:
if isinstance(value, list | tuple):
return _meta_str(value[0]) if value else None
if isinstance(value, str):
return value
return None
def _codes_from_runtime_info(runtime_info: dict[str, Any] | None) -> torch.Tensor | None:
"""Extract the ``[frames, 4]`` codec-id payload from the connector info."""
if not isinstance(runtime_info, dict):
return None
codes = runtime_info.get("codes")
if not isinstance(codes, dict):
return None
audio = codes.get("audio")
if isinstance(audio, torch.Tensor):
return audio
if isinstance(audio, list | tuple) and audio:
return torch.as_tensor(audio, dtype=torch.long)
return None
class AudexXCodec1(nn.Module):
"""Stage-1 model for Audex TTA (GenerationModelRunner)."""
input_modalities = "audio"
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
from transformers import AutoConfig, AutoModel
config = AutoConfig.from_pretrained(self.model_path, trust_remote_code=True)
self.codec = AutoModel.from_config(config, trust_remote_code=True)
self.codec.eval()
self._sample_rate = int(getattr(config, "sampling_rate", 16000))
self._max_frames = int(getattr(config, "max_tta_frames", _DEFAULT_MAX_TTA_FRAMES))
# -------------------- vLLM model interface --------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
# This stage ignores token embeddings; keep a stable dummy for the runner.
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
return None
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
state_dict = dict(weights)
self.codec.load_state_dict(state_dict, strict=True)
device = self.vllm_config.device_config.device
self.codec.to(device=device, dtype=torch.float32)
return {f"codec.{name}" for name in state_dict}
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
# The runner may unpack the NamedTuple in transit; rebuild it.
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
return OmniOutput(*model_outputs)
raise TypeError(f"AudexXCodec1 expected OmniOutput, got {type(model_outputs)}")
# -------------------- decoding --------------------
def _decode_codes(self, req_id: str | None, codes: torch.Tensor) -> torch.Tensor:
"""Decode a ``[frames, 4]`` codec-id payload to a mono waveform."""
if codes.numel() == 0:
# Zero codec tokens for the whole request. Do NOT raise: an
# exception in forward() kills the engine core for every request.
# Returning empty audio lets the serving layer fail this request.
logger.error("Audex TTA: request %s finished with no codec tokens; no audio generated", req_id)
return torch.empty(0, dtype=torch.float32)
if codes.ndim != 2 or codes.shape[1] != XCODEC1_NUM_CODEBOOKS:
logger.error(
"Audex TTA: request %s payload has shape %s, expected [frames, %d]; no audio generated",
req_id,
tuple(codes.shape),
XCODEC1_NUM_CODEBOOKS,
)
return torch.empty(0, dtype=torch.float32)
device = self.vllm_config.device_config.device
with torch.inference_mode():
frames = codes[: self._max_frames].to(device=device, dtype=torch.long)
# [frames, 4] codec ids -> [1, 4, frames] per-layer indices.
layer_offsets = (
torch.arange(XCODEC1_NUM_CODEBOOKS, device=frames.device, dtype=frames.dtype) * XCODEC1_CODEBOOK_SIZE
)
per_layer = (frames - layer_offsets.view(1, -1)).clamp(0, XCODEC1_CODEBOOK_SIZE - 1)
stacked = per_layer.transpose(0, 1).unsqueeze(0)
decoded = self.codec.decode(stacked)
audio = decoded.audio_values if hasattr(decoded, "audio_values") else decoded[0]
return audio.reshape(-1).to(dtype=torch.float32, device="cpu")
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
runtime_infos = runtime_additional_information or []
seq_token_counts = kwargs.get("seq_token_counts")
if seq_token_counts is not None:
num_reqs = len(seq_token_counts)
elif input_ids is not None and input_ids.numel() > 0 and not runtime_infos:
num_reqs = 1
else:
num_reqs = len(runtime_infos)
audios: list[torch.Tensor] = []
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
for i in range(num_reqs):
info = runtime_infos[i] if i < len(runtime_infos) else {}
info = info if isinstance(info, dict) else {}
meta = info.get("meta", {})
meta = meta if isinstance(meta, dict) else {}
req_id = _meta_str(meta.get("req_id"))
codes = _codes_from_runtime_info(info)
if codes is None:
audios.append(torch.empty(0, dtype=torch.float32))
continue
audios.append(self._decode_codes(req_id, codes))
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": [sr_tensor] * num_reqs},
)
@@ -0,0 +1,567 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
# inference_scripts_vllm/audiogen_scripts/cfg_logits_processor.py
# inference_scripts_vllm/audiogen_scripts/vllm_cfg_patch.py
# Divergences from the official files are commented at the divergence site.
"""Classifier-free guidance for the Audex thinker stage.
CFG pairs a conditional request with an unconditional (null-prompt)
companion in the same engine and blends their logits every step:
blended = uncond + cfg_scale * (cond - uncond)
Both rows receive the blended logits and the sampled token is copied from
the cond row to the uncond row, so the two sequences stay token-identical.
Requests opt in via ``SamplingParams.extra_args``:
cond: {"cfg_scale": 1.5, "cfg_role": "cond", "cfg_pair_id": <id>}
uncond: {"cfg_scale": 1.5, "cfg_role": "uncond", "cfg_pair_id": <id>}
Blending is only correct when both pair members decode the same position in
the same engine step, so ``apply_cfg_patches()`` makes the vLLM scheduler
pair-aware: a lone member waits for its partner, partners stay adjacent,
their ``num_computed_tokens`` are equalized after every schedule, and they
finish together. The patch is a no-op for engines that never see a CFG
request (the pair registry stays empty).
"""
from __future__ import annotations
from collections import deque
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.v1.sample.logits_processor import (
BatchUpdate,
LogitsProcessor,
MoveDirectionality,
)
logger = init_logger(__name__)
_COND = "cond"
_UNCOND = "uncond"
def _cfg_extra(request: Any, key: str) -> Any:
sampling_params = getattr(request, "sampling_params", None)
extra_args = getattr(sampling_params, "extra_args", None)
if extra_args:
return extra_args.get(key)
return None
class AudexCFGLogitsProcessor(LogitsProcessor):
"""Blend cond/uncond logits for classifier-free guidance.
Pairs are matched by ``cfg_pair_id``. For each pair the blended logits
are written to *both* rows so the sampler picks the same token; the
post-sampling copy of the cond token into the uncond slot is installed
by :meth:`_ensure_sample_patched` (which must run inside each worker
process, hence from ``__init__`` rather than a main-process patch).
"""
_sample_patched = False
@classmethod
def validate_params(cls, params: SamplingParams) -> None:
extra_args = params.extra_args
if not extra_args:
return
role = extra_args.get("cfg_role")
if role is not None and role not in (_COND, _UNCOND):
raise ValueError(f"cfg_role must be 'cond' or 'uncond', got '{role}'")
scale = extra_args.get("cfg_scale")
if scale is not None and (not isinstance(scale, int | float) or scale < 1.0):
raise ValueError(f"cfg_scale must be >= 1.0, got {scale}")
def __init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool) -> None:
self._info: dict[int, dict[str, Any]] = {}
self._output_tokens: dict[int, list[int]] = {}
self._pairs: list[tuple[int, int, float]] = []
self._dirty = True
self._ensure_sample_patched()
@classmethod
def _ensure_sample_patched(cls) -> None:
"""Install the post-sampling cond→uncond token copy (once per process)."""
if cls._sample_patched:
return
cls._sample_patched = True
# Divergence from the official file: it patches vLLM's
# ``GPUModelRunner._sample``, but vllm-omni's AR runner overrides
# ``_sample``, so the base-class patch would never run for the Audex
# thinker stage. Patch the omni AR runner instead.
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
orig_sample = GPUARModelRunner._sample
def _sample_with_cfg_sync(self, logits, spec_decode_metadata):
sampler_output = orig_sample(self, logits, spec_decode_metadata)
for proc in self.input_batch.logitsprocs.all:
if isinstance(proc, AudexCFGLogitsProcessor) and proc._pairs:
sampled = sampler_output.sampled_token_ids
num_rows = sampled.shape[0] if hasattr(sampled, "shape") else len(sampled)
for cond_idx, uncond_idx, _ in proc._pairs:
# Steps that sample fewer rows than the persistent
# batch (pure-prefill / partially scheduled steps)
# have nothing to sync for this pair; the scheduler
# pair patches keep both members step-locked on the
# decode steps that matter.
if cond_idx < num_rows and uncond_idx < num_rows:
sampled[uncond_idx] = sampled[cond_idx]
break
return sampler_output
_sample_with_cfg_sync._audex_cfg_sync = True # type: ignore[attr-defined]
GPUARModelRunner._sample = _sample_with_cfg_sync
logger.info("AudexCFGLogitsProcessor: patched GPUARModelRunner._sample for CFG token sync")
def is_argmax_invariant(self) -> bool:
return False
def _reset(self) -> None:
self._info.clear()
self._output_tokens.clear()
self._pairs.clear()
self._dirty = True
def update_state(self, batch_update: BatchUpdate | None) -> None:
if batch_update is None:
return
for idx in batch_update.removed:
self._info.pop(idx, None)
self._output_tokens.pop(idx, None)
if not self._info and batch_update.added:
self._reset()
for idx, params, _, output_token_ids in batch_update.added:
extra_args = params.extra_args if params else None
if extra_args and extra_args.get("cfg_role") in (_COND, _UNCOND):
self._info[idx] = {
"role": extra_args["cfg_role"],
"cfg_scale": float(extra_args.get("cfg_scale", 1.0)),
"pair_id": extra_args.get("cfg_pair_id"),
}
self._output_tokens[idx] = output_token_ids
else:
self._info.pop(idx, None)
self._output_tokens.pop(idx, None)
self._dirty = True
if self._info:
for src, dst, direction in batch_update.moved:
src_info = self._info.pop(src, None)
dst_info = self._info.pop(dst, None)
src_tokens = self._output_tokens.pop(src, None)
dst_tokens = self._output_tokens.pop(dst, None)
if src_info is not None:
self._info[dst] = src_info
if src_tokens is not None:
self._output_tokens[dst] = src_tokens
if direction == MoveDirectionality.SWAP:
if dst_info is not None:
self._info[src] = dst_info
if dst_tokens is not None:
self._output_tokens[src] = dst_tokens
self._dirty = True
def _rebuild_pairs(self) -> None:
by_pair: dict[str, dict[str, tuple[int, float]]] = {}
for idx, info in self._info.items():
pair_id = info.get("pair_id")
if pair_id is None:
continue
by_pair.setdefault(pair_id, {})[info["role"]] = (idx, info["cfg_scale"])
self._pairs = [
(roles[_COND][0], roles[_UNCOND][0], roles[_COND][1])
for roles in by_pair.values()
if _COND in roles and _UNCOND in roles
]
self._dirty = False
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if not self._info:
return logits
if self._dirty:
self._rebuild_pairs()
num_rows = logits.shape[0]
for cond_idx, uncond_idx, cfg_scale in self._pairs:
# Rows beyond this step's logits (pure-prefill / partially
# scheduled steps) have nothing to blend; decode steps carry
# both pair rows thanks to the scheduler pair patches.
if cond_idx >= num_rows or uncond_idx >= num_rows:
continue
blended = logits[uncond_idx] + cfg_scale * (logits[cond_idx] - logits[uncond_idx])
logits[cond_idx] = blended
logits[uncond_idx] = blended
return logits
# ---------------------------------------------------------------------------
# Scheduler pair synchronization
# ---------------------------------------------------------------------------
def _wait_queues(scheduler: Any) -> list[Any]:
# Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks
# structurally not-ready requests in a second ``skipped_waiting`` queue;
# pair-hold must cover both queues or a lone member parked there would
# be admitted without its partner.
queues = [scheduler.waiting]
skipped = getattr(scheduler, "skipped_waiting", None)
if skipped is not None:
queues.append(skipped)
return queues
def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None = None) -> bool:
pair_id = scheduler._cfg_req_to_pair.get(request_id)
if pair_id is None:
return True
roles = scheduler._cfg_pairs.get(pair_id, {})
if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()):
return False
if blocked_ids:
# A partner parked in skipped_waiting (e.g. waiting for remote KVs)
# may fail promotion this step while this member gets scheduled from
# waiting — hold this member until the partner is schedulable.
for rid in roles.values():
if rid != request_id and rid in blocked_ids:
return False
return True
# Schedule steps a lone pair member may wait for its partner. Partners arrive
# within a handful of steps in practice; exhausting this budget means the
# companion was never created (e.g. prompt expansion failed), and the request
# is released to decode unguided instead of hanging forever.
_MAX_PAIR_HOLD_STEPS = 512
def _drop_stale_pair(scheduler: Any, request_id: str) -> None:
pair_id = scheduler._cfg_req_to_pair.get(request_id)
if pair_id is None:
return
for rid in scheduler._cfg_pairs.pop(pair_id, {}).values():
scheduler._cfg_req_to_pair.pop(rid, None)
logger.warning(
"CFG pair %s: partner of %s never arrived after %d schedule steps; releasing the request unguided",
pair_id,
request_id,
_MAX_PAIR_HOLD_STEPS,
)
def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]:
"""Pull CFG requests whose partner is not yet admitted out of the wait queues.
The engine schedules as soon as the first member of a pair arrives over
IPC; without the hold, the lone member prefills one step early and the
pair is permanently offset by one token. Held requests are re-prepended
after the schedule step.
"""
hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {}
scheduler._cfg_hold_counts = hold_counts
skipped = getattr(scheduler, "skipped_waiting", None)
blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set()
held: list[tuple[Any, Any]] = []
for queue in _wait_queues(scheduler):
# A member is only "blocked" for its partner's sake when it sits in
# the OTHER queue; members of the queue being scanned move together.
partner_blockers = blocked_ids if queue is not skipped else set()
held_here = []
for request in list(queue):
if _pair_complete(scheduler, request.request_id, partner_blockers):
hold_counts.pop(request.request_id, None)
continue
count = hold_counts.get(request.request_id, 0) + 1
if count > _MAX_PAIR_HOLD_STEPS:
hold_counts.pop(request.request_id, None)
_drop_stale_pair(scheduler, request.request_id)
continue
hold_counts[request.request_id] = count
held_here.append(request)
if held_here:
queue.remove_requests(held_here)
held.extend((queue, req) for req in held_here)
return held
def _release_held(held: list[tuple[Any, Any]]) -> None:
for queue, request in reversed(held):
queue.prepend_request(request)
def _reorder_waiting_for_cfg(scheduler: Any) -> None:
"""Move CFG pair partners adjacent in the FCFS waiting queue."""
waiting = scheduler.waiting
# Only the FCFS queue (a deque) has caller-controlled ordering; priority
# queues order themselves and pair sync then relies on hold + equalize.
if not isinstance(waiting, deque) or len(waiting) < 2:
return
requests = list(waiting)
waiting.clear()
seen: set[str] = set()
result: list[Any] = []
for request in requests:
rid = request.request_id
if rid in seen:
continue
seen.add(rid)
result.append(request)
pair_id = scheduler._cfg_req_to_pair.get(rid)
if pair_id is None:
continue
for partner_id in scheduler._cfg_pairs.get(pair_id, {}).values():
if partner_id != rid and partner_id not in seen:
for candidate in requests:
if candidate.request_id == partner_id:
seen.add(partner_id)
result.append(candidate)
break
waiting.extend(result)
def _equalize_cfg_pair_progress(scheduler: Any, scheduler_output: Any) -> None:
"""Bring both members of every CFG pair to the same ``num_computed_tokens``.
Prefix-cache hits or unequal chunked-prefill budget can leave one member
ahead after a schedule step; reduce the faster member's allocation so
both land on the same position. Over-allocated KV blocks remain and are
consumed next step — nothing leaks.
"""
for roles in scheduler._cfg_pairs.values():
cond_id = roles.get(_COND)
uncond_id = roles.get(_UNCOND)
if not cond_id or not uncond_id:
continue
cond_sched = scheduler_output.num_scheduled_tokens.get(cond_id, 0)
uncond_sched = scheduler_output.num_scheduled_tokens.get(uncond_id, 0)
if cond_sched == 0 or uncond_sched == 0:
continue
cond_req = scheduler.requests.get(cond_id)
uncond_req = scheduler.requests.get(uncond_id)
if not cond_req or not uncond_req:
continue
if cond_req.num_computed_tokens == uncond_req.num_computed_tokens:
continue
target = min(cond_req.num_computed_tokens, uncond_req.num_computed_tokens)
feasible = all(
req.num_computed_tokens - target < sched
for req, sched in ((cond_req, cond_sched), (uncond_req, uncond_sched))
)
if not feasible:
continue
for req_id, req, orig_sched in (
(cond_id, cond_req, cond_sched),
(uncond_id, uncond_req, uncond_sched),
):
diff = req.num_computed_tokens - target
if diff > 0:
req.num_computed_tokens = target
scheduler_output.num_scheduled_tokens[req_id] = orig_sched - diff
scheduler_output.total_num_scheduled_tokens -= diff
logger.debug(
"CFG equalize %s: %d -> %d scheduled, computed -> %d",
req_id,
orig_sched,
orig_sched - diff,
target,
)
def _drop_split_pairs(scheduler: Any) -> None:
"""Deregister a desynced CFG pair (one member preempted / left behind).
Manual re-preemption surgery is unsafe against the upstream scheduler.
Instead, a pair that split anyway (KV-pressure preemption of exactly one
member, or unrecoverable progress skew) is dropped from the registry:
the cond request continues UNGUIDED (its logits are never blended again)
rather than being re-paired against a desynced partner, and the
companion's discarded output no longer matters.
"""
if not scheduler._cfg_pairs:
return
running_ids = {req.request_id for req in scheduler.running}
to_drop: list[tuple[str, str, str, str]] = []
for pair_id, roles in scheduler._cfg_pairs.items():
cond_id = roles.get(_COND)
uncond_id = roles.get(_UNCOND)
if cond_id is None or uncond_id is None:
continue
if cond_id not in scheduler.requests or uncond_id not in scheduler.requests:
continue
cond_running = cond_id in running_ids
uncond_running = uncond_id in running_ids
if cond_running != uncond_running:
# Split only counts once decoding started: a lone member that has
# computed tokens while its partner was pushed back is desynced.
runner = scheduler.requests.get(cond_id if cond_running else uncond_id)
if runner is not None and runner.num_computed_tokens > 0:
to_drop.append((pair_id, cond_id, uncond_id, "one member preempted"))
continue
if cond_running and uncond_running:
cond_req = scheduler.requests.get(cond_id)
uncond_req = scheduler.requests.get(uncond_id)
if cond_req and uncond_req and cond_req.num_computed_tokens != uncond_req.num_computed_tokens:
logger.warning(
"CFG pair progress mismatch (equalize will retry): %s@%d vs %s@%d",
cond_id,
cond_req.num_computed_tokens,
uncond_id,
uncond_req.num_computed_tokens,
)
for pair_id, cond_id, uncond_id, reason in to_drop:
scheduler._cfg_pairs.pop(pair_id, None)
scheduler._cfg_req_to_pair.pop(cond_id, None)
scheduler._cfg_req_to_pair.pop(uncond_id, None)
logger.warning(
"CFG pair %s split (%s); releasing cond=%s unguided, uncond=%s output is discarded",
pair_id,
reason,
cond_id,
uncond_id,
)
_patches_applied = False
def cfg_patches_applied() -> bool:
"""True once :func:`apply_cfg_patches` has patched the scheduler in this process."""
return _patches_applied
def _patch_scheduler() -> None:
from vllm.v1.core.sched.scheduler import Scheduler
orig_init = Scheduler.__init__
orig_add = Scheduler.add_request
orig_finish = Scheduler.finish_requests
orig_schedule = Scheduler.schedule
def _init(self, *args: Any, **kwargs: Any) -> None:
orig_init(self, *args, **kwargs)
self._cfg_pairs: dict[str, dict[str, str]] = {}
self._cfg_req_to_pair: dict[str, str] = {}
def _add(self, request: Any) -> None:
orig_add(self, request)
pair_id = _cfg_extra(request, "cfg_pair_id")
role = _cfg_extra(request, "cfg_role")
if pair_id and role:
self._cfg_pairs.setdefault(pair_id, {})[role] = request.request_id
self._cfg_req_to_pair[request.request_id] = pair_id
# An asymmetric prefix-cache hit (the cond prompt shares cached
# branches; the <unk> null prompt does not) can advance one
# member past what post-schedule equalization can claw back.
# Skip prefix-cache reads for CFG requests entirely.
if hasattr(request, "skip_reading_prefix_cache"):
request.skip_reading_prefix_cache = True
def _finish(self, request_ids: Any, finished_status: Any) -> Any:
if request_ids is None:
result = orig_finish(self, request_ids, finished_status)
self._cfg_pairs.clear()
self._cfg_req_to_pair.clear()
return result
if isinstance(request_ids, str):
request_ids = {request_ids}
else:
request_ids = set(request_ids)
# Finish both members together: a surviving partner would decode
# unpaired garbage (its logits are never blended again).
partner_ids: set[str] = set()
for req_id in request_ids:
pair_id = self._cfg_req_to_pair.get(req_id)
if pair_id is None:
continue
for rid in self._cfg_pairs.get(pair_id, {}).values():
if rid != req_id and rid in self.requests:
partner_ids.add(rid)
all_ids = request_ids | partner_ids
result = orig_finish(self, all_ids, finished_status)
for req_id in all_ids:
pair_id = self._cfg_req_to_pair.pop(req_id, None)
if pair_id:
self._cfg_pairs.pop(pair_id, None)
return result
def _schedule(self, *args: Any, **kwargs: Any) -> Any:
if not self._cfg_pairs:
return orig_schedule(self, *args, **kwargs)
_reorder_waiting_for_cfg(self)
held = _hold_incomplete_pairs(self)
# Chunked prefill can split one pair member's prompt across steps
# while the other fits in a single step; halving the long-prefill
# threshold makes both chunk, and equalize aligns the remainder.
scheduler_config = getattr(self, "scheduler_config", None)
orig_threshold = getattr(scheduler_config, "long_prefill_token_threshold", None)
if scheduler_config is not None and orig_threshold is not None:
scheduler_config.long_prefill_token_threshold = self.max_num_scheduled_tokens // 2
try:
scheduler_output = orig_schedule(self, *args, **kwargs)
finally:
if scheduler_config is not None and orig_threshold is not None:
scheduler_config.long_prefill_token_threshold = orig_threshold
_release_held(held)
_equalize_cfg_pair_progress(self, scheduler_output)
_drop_split_pairs(self)
return scheduler_output
_schedule._audex_cfg_patched = True # type: ignore[attr-defined]
Scheduler.__init__ = _init
Scheduler.add_request = _add
Scheduler.finish_requests = _finish
Scheduler.schedule = _schedule
def apply_cfg_patches() -> None:
"""Make the vLLM v1 scheduler CFG-pair-aware. Idempotent, per process.
Must run in the engine-core process before ``Scheduler`` is constructed
(the ``__init__`` wrapper installs the pair registry).
"""
global _patches_applied
if _patches_applied:
return
_patches_applied = True
logger.info("Applying Audex CFG scheduler patches to vLLM v1")
_patch_scheduler()
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Audex checkpoint layout preparation.
The nvidia/Nemotron-Labs-Audex-2B repo deduplicates weights: the
``checkpoint_folder_audiogen`` (and ``checkpoint_folder_textonly``) index
files reference safetensors shards that physically live only under
``checkpoint_folder_full``. The official repo ships a prepare script that
symlinks them; this replicates that step automatically so users can pass the
repo root without any manual preparation.
"""
import json
import os
import shutil
from vllm.logger import init_logger
logger = init_logger(__name__)
_WEIGHT_SOURCE_FOLDER = "checkpoint_folder_full"
# Download profiles: each deployment pulls only the repo subset its stages
# need. The dedup weight shards are appended per-model at call time from
# the audiogen index (see _dedup_shard_patterns); the static lists here
# hold only the model-independent folders.
_SNAPSHOT_PROFILE_PATTERNS = {
"tts": [
"config.json",
"checkpoint_folder_audiogen/*",
"audex_causal_speech_decoder/*",
],
# TTA reuses the audiogen thinker; waveform decode happens in the
# external XCodec1 checkpoint (see ensure_xcodec1_snapshot), so the
# bundled speech decoder is not pulled.
"tta": [
"config.json",
"checkpoint_folder_audiogen/*",
],
# The audio-capable full checkpoint: both weight shards, audio
# preprocessor assets, chat template, and remote-code modeling files.
"full": [
"config.json",
f"{_WEIGHT_SOURCE_FOLDER}/*",
],
}
XCODEC1_DEFAULT_REPO = "hf-audio/xcodec-hubert-general-balanced"
_AUDIOGEN_INDEX_FILE = "checkpoint_folder_audiogen/model.safetensors.index.json"
def _download_audiogen_index(model: str) -> str:
"""Resolve the audiogen dedup index to a local path (tiny, cache-first).
Cache-first keeps warm-cache boots offline-safe and avoids a Hub
metadata round trip per stage-engine boot; a cache miss falls back to
the network.
"""
from huggingface_hub import hf_hub_download
try:
return hf_hub_download(model, _AUDIOGEN_INDEX_FILE, local_files_only=True)
except Exception:
return hf_hub_download(model, _AUDIOGEN_INDEX_FILE)
def _dedup_shard_patterns(model: str) -> list[str]:
"""Shards under the full checkpoint referenced by the audiogen index.
The audiogen folder deduplicates its LM weights into
``checkpoint_folder_full`` shards (1 of 2 on the 2B, a subset of 14 on
the 30B-A3B); the ``tts``/``tta`` profiles must download exactly those.
Falls back to every full-checkpoint shard when the index is unreadable.
"""
try:
with open(_download_audiogen_index(model)) as f:
weight_map = json.load(f).get("weight_map", {})
shards = sorted(set(weight_map.values()))
if shards:
return [f"{_WEIGHT_SOURCE_FOLDER}/{shard}" for shard in shards]
except Exception as exc:
logger.warning(
"Could not resolve the audiogen dedup shards for %s from %s (%s); downloading all full-checkpoint shards",
model,
_AUDIOGEN_INDEX_FILE,
exc,
)
return [f"{_WEIGHT_SOURCE_FOLDER}/model-*.safetensors"]
def ensure_audex_snapshot(model: str, profile: str = "tts") -> str:
"""Resolve ``model`` to a local repo-root directory, downloading if needed.
Local paths pass through untouched. For an HF repo id, download (or reuse
from cache) exactly the subset the requesting stage's ``profile`` needs,
so per-stage ``model_subdir`` joins land on a real snapshot instead of
the repo-id string on a fresh cache. Falls back to a cached snapshot
when offline.
"""
if os.path.isdir(model):
return model
patterns = _SNAPSHOT_PROFILE_PATTERNS.get(profile)
if patterns is None:
raise ValueError(
f"Unknown Audex snapshot profile {profile!r}; expected one of {sorted(_SNAPSHOT_PROFILE_PATTERNS)}"
)
patterns = list(patterns)
if profile in ("tts", "tta"):
patterns += _dedup_shard_patterns(model)
from huggingface_hub import snapshot_download
try:
return snapshot_download(model, allow_patterns=patterns)
except Exception as download_exc:
try:
return snapshot_download(model, allow_patterns=patterns, local_files_only=True)
except Exception:
raise RuntimeError(
f"Could not resolve the Audex repo {model!r} (profile {profile!r}): the download "
"failed and no cached snapshot with the required folders was found."
) from download_exc
def ensure_xcodec1_snapshot(model: str | None) -> str:
"""Resolve the external XCodec1 checkpoint (TTA waveform decoder).
Accepts a local directory or an HF repo id; ``None``/empty falls back to
the official ``hf-audio/xcodec-hubert-general-balanced``.
"""
model = model or XCODEC1_DEFAULT_REPO
if os.path.isdir(model):
return model
from huggingface_hub import snapshot_download
try:
return snapshot_download(model)
except Exception as download_exc:
try:
return snapshot_download(model, local_files_only=True)
except Exception:
raise RuntimeError(
f"Could not resolve the XCodec1 checkpoint {model!r}: the download failed and "
"no cached snapshot was found. Download it manually or point the TTA deploy "
"yaml's stage-1 model at a local copy."
) from download_exc
def ensure_audiogen_weights(model_dir: str) -> None:
"""Link index-referenced shards missing from ``model_dir`` from the full checkpoint.
No-op when the shards are already present (or the layout is unexpected);
raises only if a referenced shard exists nowhere.
"""
index_path = os.path.join(model_dir, "model.safetensors.index.json")
if not os.path.isfile(index_path):
return
try:
with open(index_path) as f:
weight_map = json.load(f).get("weight_map", {})
except (OSError, ValueError) as exc:
logger.warning("Could not read %s: %s", index_path, exc)
return
source_dir = os.path.join(os.path.dirname(os.path.abspath(model_dir)), _WEIGHT_SOURCE_FOLDER)
for shard in sorted(set(weight_map.values())):
dst = os.path.join(model_dir, shard)
if os.path.exists(dst):
continue
src = os.path.join(source_dir, shard)
if not os.path.exists(src):
raise FileNotFoundError(
f"Audex checkpoint shard {shard!r} is missing from {model_dir} and "
f"{source_dir}. Download the repo's {_WEIGHT_SOURCE_FOLDER}/ folder "
"(it holds the deduplicated weight shards)."
)
try:
os.symlink(os.path.relpath(src, model_dir), dst)
logger.info("Linked Audex weight shard %s -> %s", dst, src)
except OSError:
shutil.copy2(src, dst)
logger.info("Copied Audex weight shard %s -> %s", src, dst)
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Audex (Nemotron-Labs-Audex-2B) TTS pipeline topology.
Stage 0: Thinker — ChatML text prompt → <speechcodec_N> tokens (LLM AR).
Stage 1: Code2Wav — streaming causal speech decoder → 16 kHz waveform.
Users pass the HF repo ROOT (its manifest config.json carries
``model_type: nemotron_labs_audex``); per-stage checkpoints resolve through
``model_subdir``/``tokenizer_subdir``. The decoder subfolder has no tokenizer
files, so stage 1's tokenizer also points at the thinker checkpoint.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.audex"
# <speechgen_end> token id in checkpoint_folder_audiogen's tokenizer; pinned
# by a unit test against the real tokenizer fixture.
AUDEX_SPEECHGEN_END_TOKEN_ID = 131076
# <audiogen_end> stops TTA generation (pinned in tests via models/audex/tta.py).
AUDEX_AUDIOGEN_END_TOKEN_ID = 131074
# No pipeline-level model_arch: stage-0 classes resolve from each
# checkpoint's own ``architectures`` (dense on the 2B, NemotronH on the
# 30B-A3B); forcing one name via hf_overrides would misbind the other
# size. Decoder stages keep their explicit per-stage model_arch.
AUDEX_TTS_PIPELINE = PipelineConfig(
model_type="audex_tts",
default_deploy_config_name="audex_tts.yaml",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audex_thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
model_subdir="checkpoint_folder_audiogen",
tokenizer_subdir="checkpoint_folder_audiogen",
# Emits the unconditional CFG companion for cfg_scale > 1.0
# requests; a no-op for unguided requests.
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
async_chunk_process_next_stage_input_func=(f"{_PROC}.thinker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.thinker2code2wav_full_payload",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [AUDEX_SPEECHGEN_END_TOKEN_ID],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="audex_code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="AudexCode2Wav",
model_subdir="audex_causal_speech_decoder",
tokenizer_subdir="checkpoint_folder_audiogen",
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
)
# Text-to-audio (caption → general audio): same audiogen thinker checkpoint,
# but the output rides the interleaved 4-codebook <audiocodec_N> RVQ block
# and decodes through the EXTERNAL XCodec1 checkpoint. XCodec1 is a CNN codec
# decoded over the full sequence, so this pipeline is sync full-payload only
# (deploy yaml sets async_chunk: false). CFG is effectively mandatory for TTA
# quality (official default scale 3.0).
# Cascaded speech-to-speech: the audio-capable full checkpoint serves all
# three official passes (ASR → chat → TTS) in one deployment; only the TTS
# pass (output modality "audio", final_stage_id 1) traverses the streaming
# speech decoder — text-modality passes finish at stage 0 and never reach
# the codec path. The cascade itself is orchestrated by the client/example.
AUDEX_S2S_PIPELINE = PipelineConfig(
model_type="audex_s2s",
default_deploy_config_name="audex_s2s.yaml",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audex_omni",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="latent",
model_subdir="checkpoint_folder_full",
tokenizer_subdir="checkpoint_folder_full",
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
async_chunk_process_next_stage_input_func=(f"{_PROC}.thinker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.thinker2code2wav_full_payload",
# <speechgen_end> additionally stops the TTS pass; text passes
# stop at their natural eos first. detokenize stays on for the
# text-final passes.
sampling_constraints={
"detokenize": True,
"stop_token_ids": [AUDEX_SPEECHGEN_END_TOKEN_ID],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="audex_code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="AudexCode2Wav",
model_subdir="audex_causal_speech_decoder",
# The decoder folder has no tokenizer files. Borrow the audiogen
# tokenizer (not checkpoint_folder_full): this stage resolves its
# snapshot with the "tts" download profile, which guarantees
# checkpoint_folder_audiogen but not the full folder's tokenizer,
# so per-stage / clean-cache launches would otherwise fail.
tokenizer_subdir="checkpoint_folder_audiogen",
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
)
# Audio understanding (ASR / audio QA): single-stage deployment of the
# audio-capable full checkpoint — speech (+ text instruction) in, text out.
# Modeled on the ming_flash_omni / qwen2_5_omni thinker-only pipelines.
AUDEX_THINKER_ONLY_PIPELINE = PipelineConfig(
model_type="audex_thinker_only",
default_deploy_config_name="audex_thinker_only.yaml",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audex_omni",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
engine_output_type="text",
model_subdir="checkpoint_folder_full",
tokenizer_subdir="checkpoint_folder_full",
sampling_constraints={"detokenize": True},
),
),
)
AUDEX_TTA_PIPELINE = PipelineConfig(
model_type="audex_tta",
default_deploy_config_name="audex_tta.yaml",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audex_tta_thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
model_subdir="checkpoint_folder_audiogen",
tokenizer_subdir="checkpoint_folder_audiogen",
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
custom_process_next_stage_input_func=f"{_PROC}.thinker2xcodec_full_payload",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [AUDEX_AUDIOGEN_END_TOKEN_ID],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="audex_xcodec",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="AudexXCodec1",
# Length placeholder for the stage-1 request; the bulk codec
# payload ships via thinker2xcodec_full_payload.
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
)
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Audex TTS/TTA prompt construction.
The Audex thinker consumes the exact ChatML prompt used by the official
inference script (``inference_scripts_vllm/audiogen_scripts/run_audio_gen_vllm.py``
in the nvidia/Nemotron-Labs-Audex-2B repo). The checkpoint's bundled chat
template opens a thinking block (``<think>\\n``) instead of the closed
``<think></think>`` + ``<speechgen_start>`` priming that TTS generation
requires, so the prompt is built from a literal template here; a unit test
pins it byte-for-byte against the official format.
``build_null_prompt`` is the unconditional-prompt counterpart used by
classifier-free guidance: the transcription is replaced with repeated
``<unk>`` tokens, iteratively adjusted so the tokenized null prompt is
exactly as long as the conditional prompt (the CFG pair must decode the
same positions).
"""
AUDEX_SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
# ``{text}`` carries its own leading whitespace: the conditional prompt uses
# " {transcription}" (matching the official script byte-for-byte), while the
# CFG null prompt inserts the bare ``<unk>`` run. ``<unk>`` is a special
# token, so a leading space would tokenize as a standalone space token and
# make some conditional lengths unreachable by whole-``<unk>`` steps.
_TTS_PROMPT_TEMPLATE = (
"<|im_start|>system\n{system_prompt}<|im_end|>\n"
"<|im_start|>user\n<|text to speech|> Generate speech for this transcription.{text}<|im_end|>\n"
"<|im_start|>assistant\n<think></think><speechgen_start>"
)
_TTA_PROMPT_TEMPLATE = (
"<|im_start|>system\n{system_prompt}<|im_end|>\n"
"<|im_start|>user\n<|text to audio|> Generate audio for this caption.{text}<|im_end|>\n"
"<|im_start|>assistant\n<think></think><audiogen_start>"
)
def build_cond_prompt(text: str) -> str:
"""Build the conditional TTS prompt for one transcription."""
text = text.strip()
if not text:
raise ValueError("Audex TTS requires non-empty input text")
return _TTS_PROMPT_TEMPLATE.format(system_prompt=AUDEX_SYSTEM_PROMPT, text=f" {text}")
def build_tta_cond_prompt(caption: str) -> str:
"""Build the conditional TTA prompt for one audio caption."""
caption = caption.strip()
if not caption:
raise ValueError("Audex TTA requires a non-empty caption")
return _TTA_PROMPT_TEMPLATE.format(system_prompt=AUDEX_SYSTEM_PROMPT, text=f" {caption}")
def _length_matched_null(cond_prompt: str, tokenizer, template: str, max_iters: int) -> str:
target_len = len(tokenizer.encode(cond_prompt))
n_unk = 1
for _ in range(max_iters):
prompt = template.format(system_prompt=AUDEX_SYSTEM_PROMPT, text="<unk>" * n_unk)
diff = target_len - len(tokenizer.encode(prompt))
if diff == 0:
return prompt
n_unk = max(1, n_unk + diff)
if diff < 0 and n_unk == 1:
break
raise ValueError(f"Could not length-match the CFG null prompt to {target_len} tokens")
def build_null_prompt(cond_prompt: str, tokenizer, max_iters: int = 64) -> str:
"""Unconditional TTS prompt for CFG (length-matched ``<unk>`` padding).
Replaces the transcription with ``<unk>`` repeated, adjusting the count
until ``tokenizer.encode`` yields exactly the conditional prompt's
length. Raises when no count matches so callers can fail the request
instead of submitting a misaligned pair.
"""
return _length_matched_null(cond_prompt, tokenizer, _TTS_PROMPT_TEMPLATE, max_iters)
def build_tta_null_prompt(cond_prompt: str, tokenizer, max_iters: int = 64) -> str:
"""Unconditional TTA prompt for CFG (length-matched ``<unk>`` padding)."""
return _length_matched_null(cond_prompt, tokenizer, _TTA_PROMPT_TEMPLATE, max_iters)
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
"""Vendored Audex causal speech decoder (Apache-2.0).
Verbatim copy of the ``audex_causal_speech_decoder`` remote-code package from
nvidia/Nemotron-Labs-Audex-2B. Vendored because transformers' dynamic module
loader resolves HuggingFace-cache blob symlinks to their real paths, which
breaks the package's relative imports when loading with trust_remote_code
directly from a hub snapshot.
"""
from .configuration_audex_causal_speech_decoder import AudexCausalSpeechDecoderConfig
from .modeling_audex_causal_speech_decoder import (
AudexCausalSpeechDecoderModel,
AudexCausalSpeechDecoderSession,
)
__all__ = [
"AudexCausalSpeechDecoderConfig",
"AudexCausalSpeechDecoderModel",
"AudexCausalSpeechDecoderSession",
]
@@ -0,0 +1,50 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from transformers import PretrainedConfig
class AudexCausalSpeechDecoderConfig(PretrainedConfig):
model_type = "audex_causal_speech_decoder"
def __init__(
self,
hidden_dim: int = 2048,
depth: int = 12,
heads: int = 32,
pos_meb_dim: int = 64,
hop_length: int = 320,
vq_dim: int = 2048,
lookahead_steps: int = 4,
sample_rate: int = 16000,
codebook_levels: list[int] | None = None,
codebook_size: int = 65536,
token_embed_dim: int = 8,
embed_tokens_from_codes: bool = True,
**kwargs,
):
super().__init__(**kwargs)
self.hidden_dim = hidden_dim
self.depth = depth
self.heads = heads
self.pos_meb_dim = pos_meb_dim
self.hop_length = hop_length
self.vq_dim = vq_dim
self.lookahead_steps = lookahead_steps
self.sample_rate = sample_rate
self.codebook_levels = codebook_levels or [4, 4, 4, 4, 4, 4, 4, 4]
self.codebook_size = codebook_size
self.token_embed_dim = token_embed_dim
self.embed_tokens_from_codes = embed_tokens_from_codes
@@ -0,0 +1,478 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from collections.abc import Iterator, Sequence
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import Tensor
from transformers import PreTrainedModel
from .configuration_audex_causal_speech_decoder import AudexCausalSpeechDecoderConfig
from .streaming_utils import load_audex_causal_speech_decoder as _load_decoder_for_remote_code
REMOTE_CODE_IMPORTS = (_load_decoder_for_remote_code,)
class RotaryPositionalEmbeddings(nn.Module):
def __init__(self, dim: int, max_seq_len: int = 4096, base: int = 10_000) -> None:
super().__init__()
self.dim = dim
self.base = base
self.max_seq_len = max_seq_len
self.rope_init()
self._rope_ready = False
def rope_init(self, device: torch.device | None = None) -> None:
theta = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=device)[: (self.dim // 2)].float() / self.dim))
self.register_buffer("theta", theta, persistent=False)
self.build_rope_cache(self.max_seq_len)
def build_rope_cache(self, max_seq_len: int = 4096) -> None:
self.max_seq_len = max_seq_len
seq_idx = torch.arange(max_seq_len, dtype=self.theta.dtype, device=self.theta.device)
idx_theta = torch.einsum("i, j -> ij", seq_idx, self.theta).float()
cache = torch.stack([torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1)
self.register_buffer("cache", cache, persistent=False)
def forward(self, x: torch.Tensor, *, input_pos: torch.Tensor | None = None) -> torch.Tensor:
seq_len = x.size(1)
needed_seq_len = seq_len if input_pos is None else int(input_pos.max().item()) + 1
if (
not getattr(self, "_rope_ready", False)
or self.theta.device != x.device
or needed_seq_len > self.cache.size(0)
):
self.rope_init(device=x.device)
if needed_seq_len > self.cache.size(0):
self.build_rope_cache(max(needed_seq_len, self.cache.size(0) * 2))
self._rope_ready = True
rope_cache = self.cache[:seq_len] if input_pos is None else self.cache[input_pos]
xshaped = x.float().reshape(*x.shape[:-1], -1, 2)
rope_cache = rope_cache.view(-1, xshaped.size(1), 1, xshaped.size(3), 2)
x_out = torch.stack(
[
xshaped[..., 0] * rope_cache[..., 0] - xshaped[..., 1] * rope_cache[..., 1],
xshaped[..., 1] * rope_cache[..., 0] + xshaped[..., 0] * rope_cache[..., 1],
],
-1,
)
return x_out.flatten(3).type_as(x)
class CausalCodecDecoderCache:
def __init__(self) -> None:
self.key_values: dict[int, tuple[Tensor, Tensor]] = {}
self.position = 0
def input_positions(self, length: int, device: torch.device) -> Tensor:
return torch.arange(self.position, self.position + length, device=device).unsqueeze(0)
def update(self, layer_idx: int, key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]:
if layer_idx in self.key_values:
prev_key, prev_value = self.key_values[layer_idx]
key = torch.cat([prev_key, key], dim=2)
value = torch.cat([prev_value, value], dim=2)
self.key_values[layer_idx] = (key, value)
return key, value
def advance(self, length: int) -> None:
self.position += length
def reset(self) -> None:
self.key_values.clear()
self.position = 0
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
norm_x = torch.mean(x**2, dim=-1, keepdim=True)
return x * torch.rsqrt(norm_x + self.eps) * self.weight
class MLP(nn.Module):
def __init__(self, dim: int) -> None:
super().__init__()
self.fc1 = nn.Linear(dim, 4 * dim, bias=False)
self.silu = nn.SiLU()
self.fc2 = nn.Linear(4 * dim, dim, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.fc2(self.silu(self.fc1(x)))
class Attention(nn.Module):
def __init__(self, dim: int, n_heads: int, rotary_embed: RotaryPositionalEmbeddings, layer_idx: int):
super().__init__()
if dim % n_heads != 0:
raise ValueError(f"dim must be divisible by n_heads, got dim={dim}, n_heads={n_heads}")
self.n_heads = n_heads
self.layer_idx = layer_idx
self.rotary_embed = rotary_embed
self.c_attn = nn.Linear(dim, 3 * dim, bias=False)
self.c_proj = nn.Linear(dim, dim, bias=False)
def forward(
self,
x: torch.Tensor,
cache: CausalCodecDecoderCache | None = None,
input_pos: Tensor | None = None,
) -> torch.Tensor:
batch_size, seq_len, _ = x.shape
qkv = self.c_attn(x)
head_dim = qkv.size(-1) // (3 * self.n_heads)
qkv = qkv.view(batch_size, seq_len, 3, self.n_heads, head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
q = self.rotary_embed(q.transpose(1, 2), input_pos=input_pos).transpose(1, 2)
k = self.rotary_embed(k.transpose(1, 2), input_pos=input_pos).transpose(1, 2)
if cache is None:
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
else:
if input_pos is None:
raise ValueError("input_pos is required when cache is set")
k, v = cache.update(self.layer_idx, k, v)
key_pos = torch.arange(k.size(2), device=x.device).view(1, 1, 1, -1)
attn_mask = key_pos <= input_pos.view(input_pos.size(0), 1, -1, 1)
y = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask)
return self.c_proj(y.transpose(1, 2).contiguous().view(batch_size, seq_len, -1))
class TransformerBlock(nn.Module):
def __init__(self, dim: int, n_heads: int, rotary_embed: RotaryPositionalEmbeddings, layer_idx: int):
super().__init__()
self.att_norm = RMSNorm(dim)
self.ffn_norm = RMSNorm(dim)
self.att = Attention(dim=dim, n_heads=n_heads, rotary_embed=rotary_embed, layer_idx=layer_idx)
self.mlp = MLP(dim=dim)
def forward(
self,
x: torch.Tensor,
cache: CausalCodecDecoderCache | None = None,
input_pos: Tensor | None = None,
) -> torch.Tensor:
x = x + self.att(self.att_norm(x), cache=cache, input_pos=input_pos)
return x + self.mlp(self.ffn_norm(x))
class PatchHead(nn.Module):
def __init__(self, dim: int, hop_length: int = 320):
super().__init__()
self.proj = nn.Linear(dim, hop_length, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = torch.tanh(self.proj(x))
return x.reshape(x.size(0), 1, -1)
class CausalVocosBackbone(nn.Module):
def __init__(
self,
hidden_dim: int = 2048,
depth: int = 12,
heads: int = 32,
pos_meb_dim: int = 64,
):
super().__init__()
rotary_embed = RotaryPositionalEmbeddings(dim=pos_meb_dim)
self.transformers = nn.ModuleList(
[
TransformerBlock(dim=hidden_dim, n_heads=heads, rotary_embed=rotary_embed, layer_idx=idx)
for idx in range(depth)
]
)
self.final_layer_norm = RMSNorm(hidden_dim)
def forward(self, x: torch.Tensor, cache: CausalCodecDecoderCache | None = None) -> torch.Tensor:
input_pos = None
if cache is not None:
input_pos = cache.input_positions(x.size(1), x.device).expand(x.size(0), -1)
for block in self.transformers:
x = block(x, cache=cache, input_pos=input_pos)
if cache is not None:
cache.advance(x.size(1))
return self.final_layer_norm(x)
class CausalCodecDecoderVocos(nn.Module):
def __init__(
self,
hidden_dim: int = 2048,
depth: int = 12,
heads: int = 32,
pos_meb_dim: int = 64,
hop_length: int = 320,
vq_dim: int = 2048,
lookahead_steps: int = 0,
):
super().__init__()
if lookahead_steps < 0:
raise ValueError(f"lookahead_steps must be >= 0, got {lookahead_steps}")
self.wav_proj = nn.Linear(hop_length, hidden_dim, bias=False)
self.fc_post_a = nn.Linear(vq_dim, hidden_dim, bias=False)
self.lookahead_steps = lookahead_steps
if lookahead_steps > 0:
self.lookahead_conv = nn.Conv1d(
hidden_dim,
hidden_dim,
kernel_size=lookahead_steps + 1,
padding=0,
groups=hidden_dim,
bias=False,
)
self.lookahead_act = nn.SiLU()
self.lookahead_proj = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False)
nn.init.zeros_(self.lookahead_proj.weight)
else:
self.lookahead_conv = None
self.lookahead_act = None
self.lookahead_proj = None
self.backbone = CausalVocosBackbone(hidden_dim, depth, heads, pos_meb_dim)
self.head = PatchHead(hidden_dim, hop_length)
def _project_tokens(self, vq_emb: torch.Tensor) -> torch.Tensor:
return self.fc_post_a(vq_emb)
def _apply_lookahead(self, x: torch.Tensor) -> torch.Tensor:
if self.lookahead_conv is None:
return x
if self.lookahead_act is None or self.lookahead_proj is None:
raise RuntimeError("lookahead modules are not initialized")
h = F.pad(x.transpose(1, 2), (0, self.lookahead_steps))
h = self.lookahead_proj(self.lookahead_act(self.lookahead_conv(h)))
return x + h.transpose(1, 2)
def _apply_lookahead_window(self, x: torch.Tensor) -> torch.Tensor:
if self.lookahead_conv is None:
return x
if self.lookahead_act is None or self.lookahead_proj is None:
raise RuntimeError("lookahead modules are not initialized")
if x.size(1) <= self.lookahead_steps:
raise ValueError(f"lookahead window needs more than {self.lookahead_steps} frames, got {x.size(1)}")
h = self.lookahead_proj(self.lookahead_act(self.lookahead_conv(x.transpose(1, 2))))
return x[:, : h.size(2)] + h.transpose(1, 2)
def decode_cached(
self,
vq_emb: torch.Tensor,
cache: CausalCodecDecoderCache,
lookahead_vq_emb: torch.Tensor | None = None,
) -> torch.Tensor:
x = self._project_tokens(vq_emb)
if self.lookahead_steps > 0:
if lookahead_vq_emb is None:
lookahead_vq_emb = vq_emb.new_zeros(vq_emb.size(0), self.lookahead_steps, vq_emb.size(-1))
if lookahead_vq_emb.size(1) != self.lookahead_steps:
raise ValueError(
f"lookahead_vq_emb must have {self.lookahead_steps} frames, got {lookahead_vq_emb.size(1)}"
)
lookahead_x = self._project_tokens(lookahead_vq_emb)
x = self._apply_lookahead_window(torch.cat([x, lookahead_x], dim=1))
x = self.backbone(x, cache=cache)
return self.head(x)
def forward(
self,
vq_emb: torch.Tensor,
patched_wav: torch.Tensor | None = None,
alpha: float = 0.0,
) -> torch.Tensor:
x = self._project_tokens(vq_emb)
x = self._apply_lookahead(x)
if patched_wav is not None:
h = self.wav_proj(patched_wav)
mask = torch.bernoulli(
torch.full(
(x.size(0), x.size(1), 1),
min(max(alpha, 0.0), 1.0),
device=x.device,
dtype=x.dtype,
)
)
x = x + h * mask
return self.head(self.backbone(x))
class AudexSpeechTokenEmbedder(nn.Module):
def __init__(
self,
output_dim: int,
token_embed_dim: int,
codebook_levels: Sequence[int],
) -> None:
super().__init__()
if len(codebook_levels) != token_embed_dim:
raise ValueError(
f"token_embed_dim={token_embed_dim} must match codebook_levels length={len(codebook_levels)}"
)
self.codebook_levels = tuple(int(level) for level in codebook_levels)
self.project_out = nn.Linear(token_embed_dim, output_dim)
def forward(self, indices: torch.Tensor) -> torch.Tensor:
if indices.size(-1) != 1:
raise ValueError(f"indices last dimension must be 1, got {indices.size(-1)}")
levels = torch.tensor(self.codebook_levels, dtype=torch.long, device=indices.device)
basis = torch.cumprod(torch.cat([levels.new_ones(1), levels[:-1]]), dim=0)
level_indices = (indices.long() // basis) % levels
dtype = self.project_out.weight.dtype
codes = level_indices.to(dtype=dtype)
levels = levels.to(dtype=dtype)
codes = codes * (2.0 / (levels - 1.0)) - 1.0
return self.project_out(codes)
def get_output_from_indices(self, indices: torch.Tensor) -> torch.Tensor:
return self(indices)
class AudexCausalSpeechDecoderModel(PreTrainedModel):
config_class = AudexCausalSpeechDecoderConfig
base_model_prefix = "module"
all_tied_weights_keys: dict[str, Any] = {}
Cache = CausalCodecDecoderCache
def __init__(self, config: AudexCausalSpeechDecoderConfig):
super().__init__(config)
self.audex_speech_token_embedder = AudexSpeechTokenEmbedder(
output_dim=config.vq_dim,
token_embed_dim=config.token_embed_dim,
codebook_levels=config.codebook_levels,
)
self.module = CausalCodecDecoderVocos(
hidden_dim=config.hidden_dim,
depth=config.depth,
heads=config.heads,
pos_meb_dim=config.pos_meb_dim,
hop_length=config.hop_length,
vq_dim=config.vq_dim,
lookahead_steps=config.lookahead_steps,
)
@property
def lookahead_steps(self) -> int:
return self.module.lookahead_steps
def create_cache(self) -> CausalCodecDecoderCache:
return CausalCodecDecoderCache()
def decode_cached(
self,
vq_emb: torch.Tensor,
cache: CausalCodecDecoderCache,
lookahead_vq_emb: torch.Tensor | None = None,
) -> torch.Tensor:
return self.module.decode_cached(vq_emb, cache, lookahead_vq_emb=lookahead_vq_emb)
def create_session(
self,
*,
chunk_frames: int = 1,
sample_rate: int | None = None,
return_numpy: bool = True,
) -> AudexCausalSpeechDecoderSession:
return AudexCausalSpeechDecoderSession(
decoder=self,
chunk_frames=chunk_frames,
sample_rate=sample_rate or self.config.sample_rate,
return_numpy=return_numpy,
)
def forward(
self,
vq_emb: torch.Tensor,
patched_wav: torch.Tensor | None = None,
alpha: float = 0.0,
) -> torch.Tensor:
return self.module(vq_emb, patched_wav=patched_wav, alpha=alpha)
class AudexCausalSpeechDecoderSession:
def __init__(
self,
decoder: AudexCausalSpeechDecoderModel,
*,
chunk_frames: int,
sample_rate: int,
return_numpy: bool,
):
if chunk_frames <= 0:
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
self.decoder = decoder
self.chunk_frames = chunk_frames
self.sample_rate = sample_rate
self.return_numpy = return_numpy
self.cache = decoder.create_cache()
self.buffer: list[list[int]] = []
@property
def device(self) -> torch.device:
return next(self.decoder.parameters()).device
def reset(self) -> None:
self.cache = self.decoder.create_cache()
self.buffer.clear()
def push(self, token_frames: Sequence[Sequence[int]]) -> Iterator[tuple[int, Any]]:
self.buffer.extend(list(frame) for frame in token_frames)
yield from self._drain(flush=False)
def flush(self) -> Iterator[tuple[int, Any]]:
yield from self._drain(flush=True)
def _drain(self, *, flush: bool) -> Iterator[tuple[int, Any]]:
ready_frames = len(self.buffer) - self.decoder.lookahead_steps
while self.buffer and (flush or ready_frames >= self.chunk_frames):
emit_frames = min(self.chunk_frames, len(self.buffer)) if flush else self.chunk_frames
wav = self._decode_buffered_frames(emit_frames, flush=flush)
del self.buffer[:emit_frames]
ready_frames = len(self.buffer) - self.decoder.lookahead_steps
yield self.sample_rate, self._format_chunk(wav)
def _embed_speech_token_frames(self, token_frames: Sequence[Sequence[int]]) -> torch.Tensor:
indices = torch.tensor(token_frames, dtype=torch.long, device=self.device).unsqueeze(0)
return self.decoder.audex_speech_token_embedder.get_output_from_indices(indices)
def _decode_buffered_frames(self, emit_frames: int, *, flush: bool) -> torch.Tensor:
with torch.inference_mode():
vq_emb = self._embed_speech_token_frames(self.buffer[:emit_frames])
lookahead_vq_emb = None
if self.decoder.lookahead_steps > 0:
future_frames = self.buffer[emit_frames : emit_frames + self.decoder.lookahead_steps]
future_parts = []
if future_frames:
future_parts.append(self._embed_speech_token_frames(future_frames))
missing_frames = self.decoder.lookahead_steps - len(future_frames) if flush else 0
if missing_frames > 0:
future_parts.append(vq_emb.new_zeros(vq_emb.size(0), missing_frames, vq_emb.size(-1)))
lookahead_vq_emb = torch.cat(future_parts, dim=1) if future_parts else None
return self.decoder.decode_cached(vq_emb, self.cache, lookahead_vq_emb=lookahead_vq_emb)
def _format_chunk(self, wav: torch.Tensor) -> Any:
chunk = wav.squeeze().float().detach().cpu()
if not self.return_numpy:
return chunk
import numpy as np
return chunk.numpy().astype(np.float32, copy=False)
@@ -0,0 +1,140 @@
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
from typing import TYPE_CHECKING, Any
import torch
from transformers import AutoModel
if TYPE_CHECKING:
from .modeling_audex_causal_speech_decoder import AudexCausalSpeechDecoderModel
def freeze_model(model: torch.nn.Module) -> torch.nn.Module:
model.eval()
for param in model.parameters():
param.requires_grad = False
return model
def load_audex_causal_speech_decoder(
model_path: str,
*,
device: torch.device | str = "cuda",
dtype: torch.dtype | str | None = None,
) -> AudexCausalSpeechDecoderModel:
kwargs = {"trust_remote_code": True}
if dtype is not None:
kwargs["torch_dtype"] = dtype
model = AutoModel.from_pretrained(model_path, **kwargs)
return freeze_model(model.to(device))
def _decoder_device(decoder: AudexCausalSpeechDecoderModel) -> torch.device:
return next(decoder.parameters()).device
def _embed_speech_token_frames(
decoder: AudexCausalSpeechDecoderModel,
token_frames: Sequence[Sequence[int]],
device: torch.device,
) -> torch.Tensor:
indices = torch.tensor(token_frames, dtype=torch.long, device=device).unsqueeze(0)
return decoder.audex_speech_token_embedder.get_output_from_indices(indices)
def decode_speech_token_frames(
token_frames: Iterable[Sequence[int]],
decoder: AudexCausalSpeechDecoderModel,
*,
device: torch.device | str | None = None,
chunk_frames: int = 1,
) -> Iterator[torch.Tensor]:
if chunk_frames <= 0:
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
device = torch.device(device) if device is not None else _decoder_device(decoder)
cache = decoder.create_cache()
buffer: list[list[int]] = []
lookahead_steps = decoder.lookahead_steps
for token_frame in token_frames:
buffer.append(list(token_frame))
while len(buffer) > lookahead_steps:
emit_frames = min(chunk_frames, len(buffer) - lookahead_steps)
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=False)
del buffer[:emit_frames]
yield wav
while buffer:
emit_frames = min(chunk_frames, len(buffer))
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=True)
del buffer[:emit_frames]
yield wav
async def decode_speech_token_stream(
token_stream: AsyncIterable[Sequence[int]],
decoder: AudexCausalSpeechDecoderModel,
*,
device: torch.device | str | None = None,
chunk_frames: int = 1,
) -> AsyncIterator[torch.Tensor]:
if chunk_frames <= 0:
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
device = torch.device(device) if device is not None else _decoder_device(decoder)
cache = decoder.create_cache()
buffer: list[list[int]] = []
lookahead_steps = decoder.lookahead_steps
async for token_frame in token_stream:
buffer.append(list(token_frame))
while len(buffer) > lookahead_steps:
emit_frames = min(chunk_frames, len(buffer) - lookahead_steps)
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=False)
del buffer[:emit_frames]
yield wav
while buffer:
emit_frames = min(chunk_frames, len(buffer))
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=True)
del buffer[:emit_frames]
yield wav
def _decode_buffered_frames(
decoder: AudexCausalSpeechDecoderModel,
cache: Any,
buffer: Sequence[Sequence[int]],
emit_frames: int,
device: torch.device,
*,
flush: bool,
) -> torch.Tensor:
with torch.inference_mode():
vq_emb = _embed_speech_token_frames(decoder, buffer[:emit_frames], device)
lookahead_vq_emb = None
if decoder.lookahead_steps > 0:
future_frames = buffer[emit_frames : emit_frames + decoder.lookahead_steps]
future_parts = []
if future_frames:
future_parts.append(_embed_speech_token_frames(decoder, future_frames, device))
missing_frames = decoder.lookahead_steps - len(future_frames) if flush else 0
if missing_frames > 0:
future_parts.append(vq_emb.new_zeros(vq_emb.size(0), missing_frames, vq_emb.size(-1)))
lookahead_vq_emb = torch.cat(future_parts, dim=1) if future_parts else None
return decoder.decode_cached(vq_emb, cache, lookahead_vq_emb=lookahead_vq_emb)
@@ -0,0 +1,293 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
# inference_scripts_vllm/audiogen_scripts/tta_rvq_logits_processor.py
# inference_scripts_vllm/audiogen_scripts/run_audio_gen_vllm_rvq_logit_mask.py
# Divergences from the official files are commented at the divergence site.
"""Audex text-to-audio (TTA) token space: RVQ phase masking and helpers.
TTA output is an interleaved 4-codebook RVQ stream over ``<audiocodec_N>``
tokens: generated position ``p`` (counted from ``<audiogen_start>``) must
come from codebook ``p % 4``, whose codec-id range is
``[phase * 1024, (phase + 1) * 1024)``. The tokenizer defines 8192
``<audiocodec_*>`` symbols; only codec ids 0..4095 (the 4 quantizers XCodec1
decodes) are generated — the upper half stays disallowed.
Unguided/unmasked sampling emits phase-invalid sequences that XCodec1 cannot
decode, so :class:`TTARVQPhaseMaskLogitsProcessor` is required for validity,
driven per request by ``SamplingParams.extra_args["tta_rvq"]``:
{
"phase_token_ids": [[...1024 ids...] x 4],
"start_tid": <audiogen_start id>,
"end_tid": <audiogen_end id>,
"codec_cap": <max codec tokens or None>,
"start_in_prompt": <bool>,
}
"""
from __future__ import annotations
import os
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.v1.sample.logits_processor import (
BatchUpdate,
LogitsProcessor,
MoveDirectionality,
)
logger = init_logger(__name__)
NEG_INF = float("-inf")
XCODEC1_NUM_CODEBOOKS = 4
XCODEC1_CODEBOOK_SIZE = 1024
# Pinned by a unit test against the real checkpoint tokenizer.
AUDEX_AUDIOGEN_START_TOKEN_ID = 131073
AUDEX_AUDIOGEN_END_TOKEN_ID = 131074
AUDEX_AUDIOCODEC_TOKEN_OFFSET = 196613
AUDEX_AUDIOCODEC_VOCAB_SIZE = 8192
def build_tta_phase_token_ids(tokenizer: Any) -> tuple[list[list[int]], int, int]:
"""Group tokenizer ids of ``<audiocodec_N>`` into the 4 RVQ phases.
Phase ``p`` collects the tokenizer ids for codec ids
``[p*1024, (p+1)*1024)``. Fails loudly on missing markers or an
incomplete codec vocab.
Returns ``(phase_token_ids, audiogen_start_tid, audiogen_end_tid)``.
"""
vocab = tokenizer.get_vocab()
start_tid = vocab.get("<audiogen_start>")
end_tid = vocab.get("<audiogen_end>")
if start_tid is None or end_tid is None:
raise ValueError(f"Missing marker token ids: audiogen_start={start_tid}, audiogen_end={end_tid}")
phase_token_ids: list[list[int]] = []
for phase in range(XCODEC1_NUM_CODEBOOKS):
ids: list[int] = []
missing: list[int] = []
for codec_id in range(phase * XCODEC1_CODEBOOK_SIZE, (phase + 1) * XCODEC1_CODEBOOK_SIZE):
tok = vocab.get(f"<audiocodec_{codec_id}>")
if tok is None:
missing.append(codec_id)
else:
ids.append(tok)
if missing:
raise ValueError(
f"Incomplete codec vocab for RVQ phase {phase}: missing {len(missing)} codec ids (e.g. {missing[:5]})"
)
phase_token_ids.append(ids)
return phase_token_ids, int(start_tid), int(end_tid)
def validate_rvq_phase(
codec_ids: list[int],
codebook_size: int = XCODEC1_CODEBOOK_SIZE,
num_codebooks: int = XCODEC1_NUM_CODEBOOKS,
) -> dict[str, Any]:
"""Check that ``codec_ids[i]`` falls in RVQ phase ``i % 4``'s codec range."""
mismatch_count = 0
first_mismatch = None
for i, codec_id in enumerate(codec_ids):
expected = i % num_codebooks
lo = expected * codebook_size
if not (lo <= codec_id < lo + codebook_size):
mismatch_count += 1
if first_mismatch is None:
first_mismatch = [i, int(codec_id), int(codec_id // codebook_size), int(expected)]
return {
"phase_valid": mismatch_count == 0,
"mismatch_count": mismatch_count,
"first_mismatch": first_mismatch,
"codec_count": len(codec_ids),
}
class TTARVQPhaseMaskLogitsProcessor(LogitsProcessor):
"""Mask logits to the RVQ-phase-valid token set for Audex TTA requests.
Per batch row:
``_req[idx]`` -> {"start_tid", "end_tid", "codec_cap", "start_in_prompt"}
``_output_tokens[idx]`` -> live reference to the request's output ids
``_start_pos[idx]`` -> cached index of <audiogen_start> once seen;
-1 means the prompt already ended with it.
The disallow masks are vocab-sized boolean tensors built once on the
first ``apply`` (using the real ``logits.shape[-1]`` so vocab padding is
also forbidden).
"""
def __init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool) -> None:
self.device = device
self._req: dict[int, dict[str, Any]] = {}
self._output_tokens: dict[int, list[int]] = {}
self._start_pos: dict[int, int] = {}
self._phase_token_ids: list[list[int]] | None = None
self._mask_end_tid: int | None = None
self._disallow_phase: torch.Tensor | None = None
self._disallow_phase0_end: torch.Tensor | None = None
self._disallow_except_end: torch.Tensor | None = None
def is_argmax_invariant(self) -> bool:
return False
def _reset(self) -> None:
self._req.clear()
self._output_tokens.clear()
self._start_pos.clear()
def update_state(self, batch_update: BatchUpdate | None) -> None:
if batch_update is None:
return
for idx in batch_update.removed:
self._req.pop(idx, None)
self._output_tokens.pop(idx, None)
self._start_pos.pop(idx, None)
if not self._req and batch_update.added:
self._reset()
for idx, params, _, output_token_ids in batch_update.added:
extra_args = params.extra_args if params else None
tta = extra_args.get("tta_rvq") if extra_args else None
if tta:
self._req[idx] = {
"start_tid": tta["start_tid"],
"end_tid": tta["end_tid"],
"codec_cap": tta.get("codec_cap"),
"start_in_prompt": bool(tta.get("start_in_prompt", False)),
}
self._output_tokens[idx] = output_token_ids
if self._req[idx]["start_in_prompt"]:
self._start_pos[idx] = -1
else:
self._start_pos.pop(idx, None)
if self._phase_token_ids is None:
self._phase_token_ids = tta["phase_token_ids"]
self._mask_end_tid = tta["end_tid"]
else:
self._req.pop(idx, None)
self._output_tokens.pop(idx, None)
self._start_pos.pop(idx, None)
if self._req:
for src, dst, direction in batch_update.moved:
src_req = self._req.pop(src, None)
dst_req = self._req.pop(dst, None)
src_tokens = self._output_tokens.pop(src, None)
dst_tokens = self._output_tokens.pop(dst, None)
src_pos = self._start_pos.pop(src, None)
dst_pos = self._start_pos.pop(dst, None)
if src_req is not None:
self._req[dst] = src_req
if src_tokens is not None:
self._output_tokens[dst] = src_tokens
if src_pos is not None:
self._start_pos[dst] = src_pos
if direction == MoveDirectionality.SWAP:
if dst_req is not None:
self._req[src] = dst_req
if dst_tokens is not None:
self._output_tokens[src] = dst_tokens
if dst_pos is not None:
self._start_pos[src] = dst_pos
def _build_masks(self, vocab_size: int) -> None:
if self._phase_token_ids is None or self._mask_end_tid is None:
raise RuntimeError(
"TTARVQPhaseMaskLogitsProcessor: phase token ids missing; "
"extra_args['tta_rvq'] was not provided for a TTA request."
)
if len(self._phase_token_ids) != XCODEC1_NUM_CODEBOOKS:
raise RuntimeError(
f"TTARVQPhaseMaskLogitsProcessor: expected {XCODEC1_NUM_CODEBOOKS} phase id "
f"lists, got {len(self._phase_token_ids)}."
)
allow = torch.zeros((XCODEC1_NUM_CODEBOOKS, vocab_size), dtype=torch.bool, device=self.device)
for phase, ids in enumerate(self._phase_token_ids):
if not ids:
raise RuntimeError(f"TTARVQPhaseMaskLogitsProcessor: empty token id list for RVQ phase {phase}.")
idx = torch.as_tensor(ids, dtype=torch.long, device=self.device)
if int(idx.max()) >= vocab_size or int(idx.min()) < 0:
raise RuntimeError(
f"TTARVQPhaseMaskLogitsProcessor: phase {phase} token id out of range for vocab_size={vocab_size}."
)
allow[phase].index_fill_(0, idx, True)
end_onehot = torch.zeros(vocab_size, dtype=torch.bool, device=self.device)
if self._mask_end_tid < 0 or self._mask_end_tid >= vocab_size:
raise RuntimeError(
f"TTARVQPhaseMaskLogitsProcessor: end token id out of range "
f"for vocab_size={vocab_size}: {self._mask_end_tid}."
)
end_onehot[self._mask_end_tid] = True
self._disallow_phase = ~allow
self._disallow_phase0_end = ~(allow[0] | end_onehot)
self._disallow_except_end = ~end_onehot
logger.info(
"TTARVQPhaseMaskLogitsProcessor: built masks vocab=%d end_tid=%d (pid=%d)",
vocab_size,
self._mask_end_tid,
os.getpid(),
)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if not self._req:
return logits
if self._disallow_phase is None:
self._build_masks(logits.shape[-1])
num_rows = logits.shape[0]
for idx, state in self._req.items():
# Rows beyond this step's logits (pure-prefill / partially
# scheduled steps) have nothing to mask; the persistent-batch
# index re-enters range on the decode steps that sample.
if idx >= num_rows:
continue
tokens = self._output_tokens.get(idx)
if tokens is None:
continue
n = len(tokens)
start_pos = self._start_pos.get(idx)
if start_pos is None:
start_tid = state["start_tid"]
for i in range(n):
if tokens[i] == start_tid:
start_pos = i
self._start_pos[idx] = i
break
if start_pos is None:
continue
position = n - start_pos - 1
if position < 0:
continue
cap = state["codec_cap"]
if cap is not None and position >= cap:
disallow = self._disallow_except_end
else:
phase = position & (XCODEC1_NUM_CODEBOOKS - 1)
if phase == 0:
disallow = self._disallow_phase0_end if position > 0 else self._disallow_phase[0]
else:
disallow = self._disallow_phase[phase]
logits[idx].masked_fill_(disallow, NEG_INF)
return logits
@@ -84,6 +84,31 @@ _OMNI_MODELS = {
"cosyvoice3",
"CosyVoice3Model",
),
"NemotronDenseForCausalLM": (
"audex",
"audex_thinker",
"NemotronDenseForCausalLM",
),
"AudexCode2Wav": (
"audex",
"audex_code2wav",
"AudexCode2Wav",
),
"AudexXCodec1": (
"audex",
"audex_xcodec",
"AudexXCodec1",
),
"NemotronDenseAudexForConditionalGeneration": (
"audex",
"audex_omni",
"NemotronDenseAudexForConditionalGeneration",
),
"NemotronHAudexForConditionalGeneration": (
"audex",
"audex_omni",
"NemotronHAudexForConditionalGeneration",
),
"OmniVoiceModel": (
"omnivoice",
"omnivoice",