[New Model] support nemotron Audex (#4976)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
64 files changed
+7760
-39
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,227 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex code2wav: streaming speech-codec-token → waveform stage.
|
||||
|
||||
Wraps the vendored Audex causal speech decoder (see ``speech_decoder/``).
|
||||
Unlike the flow/vocoder stages of other TTS models, the decoder is natively
|
||||
stateful and streaming: one ``AudexCausalSpeechDecoderSession`` per request
|
||||
holds the decode cache, each engine step pushes only the newly received codec
|
||||
frames, and the session's 4-frame lookahead tail is drained by ``flush()``
|
||||
when the request finishes. The stage therefore never re-decodes left context
|
||||
and always returns delta audio.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.audex.speech_decoder import (
|
||||
AudexCausalSpeechDecoderConfig,
|
||||
AudexCausalSpeechDecoderModel,
|
||||
AudexCausalSpeechDecoderSession,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Frames per decoder emit. 5 frames = 100 ms at 50 fps — the granularity the
|
||||
# official streaming demo uses; small enough for a fast first audible chunk.
|
||||
_DEFAULT_EMIT_CHUNK_FRAMES = 5
|
||||
|
||||
|
||||
def _meta_int(value: Any, default: int = 0) -> int:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return int(value.reshape(-1)[0].item()) if value.numel() else default
|
||||
if isinstance(value, list | tuple):
|
||||
return _meta_int(value[0], default) if value else default
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _meta_bool(value: Any) -> bool:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return bool(value.reshape(-1)[0].item()) if value.numel() else False
|
||||
if isinstance(value, list | tuple):
|
||||
return _meta_bool(value[0]) if value else False
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _meta_str(value: Any) -> str | None:
|
||||
if isinstance(value, list | tuple):
|
||||
return _meta_str(value[0]) if value else None
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _codes_from_runtime_info(runtime_info: dict[str, Any] | None) -> list[int]:
|
||||
"""Extract this step's new codec frame ids from the connector payload."""
|
||||
if not isinstance(runtime_info, dict):
|
||||
return []
|
||||
codes = runtime_info.get("codes")
|
||||
if not isinstance(codes, dict):
|
||||
return []
|
||||
audio = codes.get("audio")
|
||||
if isinstance(audio, torch.Tensor):
|
||||
return audio.reshape(-1).tolist()
|
||||
if isinstance(audio, list | tuple):
|
||||
return [int(c) for c in audio]
|
||||
return []
|
||||
|
||||
|
||||
class AudexCode2Wav(nn.Module):
|
||||
"""Stage-1 model for Audex TTS (GenerationModelRunner)."""
|
||||
|
||||
input_modalities = "audio"
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
# Build the decoder skeleton from config so vLLM's memory profiler
|
||||
# sees it at startup; weights load later in load_weights().
|
||||
config = AudexCausalSpeechDecoderConfig.from_pretrained(self.model_path)
|
||||
self.decoder = AudexCausalSpeechDecoderModel(config)
|
||||
self.decoder.eval()
|
||||
self._sample_rate = int(config.sample_rate)
|
||||
|
||||
extra_cfg = self._connector_extra_config()
|
||||
self._emit_chunk_frames = _meta_int(extra_cfg.get("decoder_emit_chunk_frames"), _DEFAULT_EMIT_CHUNK_FRAMES)
|
||||
if self._emit_chunk_frames <= 0:
|
||||
raise ValueError(f"decoder_emit_chunk_frames must be positive, got {self._emit_chunk_frames}")
|
||||
|
||||
# One streaming session per in-flight request, keyed by the request id
|
||||
# carried in each chunk payload's meta (equal to the stage-1 engine
|
||||
# request id, so on_requests_finished can free aborted requests too).
|
||||
self._sessions: dict[str, AudexCausalSpeechDecoderSession] = {}
|
||||
|
||||
def _connector_extra_config(self) -> dict[str, Any]:
|
||||
connector_cfg = getattr(self.vllm_config.model_config, "stage_connector_config", None)
|
||||
if isinstance(connector_cfg, dict):
|
||||
extra = connector_cfg.get("extra", connector_cfg)
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
extra = getattr(connector_cfg, "extra", None)
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
|
||||
# -------------------- vLLM model interface --------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
# This stage ignores token embeddings; keep a stable dummy for the runner.
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
|
||||
return None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
state_dict = dict(weights)
|
||||
self.decoder.load_state_dict(state_dict, strict=True)
|
||||
device = self.vllm_config.device_config.device
|
||||
self.decoder.to(device=device, dtype=torch.float32)
|
||||
return {f"decoder.{name}" for name in state_dict}
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: Iterable[str]) -> None:
|
||||
"""Free decoder sessions for finished/aborted requests."""
|
||||
for req_id in finished_req_ids:
|
||||
self._sessions.pop(req_id, None)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
# The runner may unpack the NamedTuple in transit; rebuild it.
|
||||
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
|
||||
return OmniOutput(*model_outputs)
|
||||
raise TypeError(f"AudexCode2Wav expected OmniOutput, got {type(model_outputs)}")
|
||||
|
||||
# -------------------- decoding --------------------
|
||||
|
||||
def _session_for(self, req_id: str) -> AudexCausalSpeechDecoderSession:
|
||||
session = self._sessions.get(req_id)
|
||||
if session is None:
|
||||
session = self.decoder.create_session(
|
||||
chunk_frames=self._emit_chunk_frames,
|
||||
return_numpy=False,
|
||||
)
|
||||
self._sessions[req_id] = session
|
||||
return session
|
||||
|
||||
def _decode_request(self, req_id: str, codes: list[int], finished: bool) -> torch.Tensor:
|
||||
"""Push new frames into the request's session; return this step's delta audio."""
|
||||
if finished and not codes and req_id not in self._sessions:
|
||||
# Zero codec tokens for the whole request. Do NOT raise: an
|
||||
# exception in forward() kills the engine core for every request.
|
||||
# Returning empty audio lets the serving layer fail this request.
|
||||
logger.error("Audex code2wav: request %s finished with no codec tokens; no audio generated", req_id)
|
||||
return torch.empty(0, dtype=torch.float32)
|
||||
chunks: list[torch.Tensor] = []
|
||||
session = self._session_for(req_id)
|
||||
with torch.inference_mode():
|
||||
if codes:
|
||||
for _sr, chunk in session.push([[int(c)] for c in codes]):
|
||||
chunks.append(chunk.reshape(-1))
|
||||
if finished:
|
||||
for _sr, chunk in session.flush():
|
||||
chunks.append(chunk.reshape(-1))
|
||||
if finished:
|
||||
self._sessions.pop(req_id, None)
|
||||
if not chunks:
|
||||
return torch.empty(0, dtype=torch.float32)
|
||||
return torch.cat(chunks).to(torch.float32)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
runtime_infos = runtime_additional_information or []
|
||||
# One output entry per scheduled request, even on steps where a
|
||||
# request's chunk payload has not arrived yet.
|
||||
seq_token_counts = kwargs.get("seq_token_counts")
|
||||
if seq_token_counts is not None:
|
||||
num_reqs = len(seq_token_counts)
|
||||
elif input_ids is not None and input_ids.numel() > 0 and not runtime_infos:
|
||||
num_reqs = 1
|
||||
else:
|
||||
num_reqs = len(runtime_infos)
|
||||
|
||||
audios: list[torch.Tensor] = []
|
||||
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
|
||||
for i in range(num_reqs):
|
||||
info = runtime_infos[i] if i < len(runtime_infos) else {}
|
||||
info = info if isinstance(info, dict) else {}
|
||||
meta = info.get("meta", {})
|
||||
meta = meta if isinstance(meta, dict) else {}
|
||||
req_id = _meta_str(meta.get("req_id"))
|
||||
# async_chunk delivers ``stream_finished`` (the adapter strips
|
||||
# ``finished``); the sync full-payload path delivers ``finished``.
|
||||
finished = _meta_bool(meta.get("stream_finished")) or _meta_bool(meta.get("finished"))
|
||||
codes = _codes_from_runtime_info(info)
|
||||
if req_id is None:
|
||||
if codes:
|
||||
raise ValueError("Audex code2wav chunk payload is missing meta.req_id")
|
||||
audios.append(torch.empty(0, dtype=torch.float32))
|
||||
continue
|
||||
audios.append(self._decode_request(req_id, codes, finished))
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audios, "sr": [sr_tensor] * num_reqs},
|
||||
)
|
||||
@@ -0,0 +1,322 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
|
||||
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/modeling_audex_vllm.py
|
||||
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/audio_encoder.py
|
||||
"""Audex audio-understanding model (``checkpoint_folder_full``): S2T / audio QA.
|
||||
|
||||
Mirrors the Qwen2-Audio pattern with the 2B dense backbone:
|
||||
* ``language_model`` = the vllm-omni ``NemotronDenseForCausalLM`` port
|
||||
(wrapped via ``init_vllm_registered_model``; resolved through the omni
|
||||
model registry).
|
||||
* ``audio_tower`` = transformers ``Qwen2AudioEncoder`` ("NV-Whisper");
|
||||
a parameter-free avg pool maps 1500 → 750 frames per padded 30 s clip.
|
||||
* ``audio_projector`` = RMSNorm → fc1 → relu² → fc2 (no bias), matching the
|
||||
release ``NemotronHAudexProjector``.
|
||||
|
||||
``embed_multimodal(...)`` runs encoder+projector and returns one embedding
|
||||
tensor per audio item; vLLM's ``SupportsMultiModal.embed_input_ids`` merges
|
||||
them into the ``<so_embedding>`` placeholder positions. The encoder runs in
|
||||
microbatches over clips so peak memory is bounded regardless of audio length
|
||||
(up to the 900 s / 30-clip cap enforced by the processor).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from vllm_omni.model_executor.models.audex.audex_omni_processing import (
|
||||
AUDIO_ENCODER_MICRO_BATCH,
|
||||
SOUND_TOKEN,
|
||||
AudexDummyInputsBuilder,
|
||||
AudexMultiModalProcessor,
|
||||
AudexProcessingInfo,
|
||||
)
|
||||
|
||||
|
||||
class AudexProjector(nn.Module):
|
||||
"""Megatron sound_projection equivalent (matches HF NemotronHAudexProjector)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_encoder_hidden_size: int,
|
||||
intermediate_size: int,
|
||||
text_hidden_size: int,
|
||||
activation: str = "relu2",
|
||||
norm_eps: float = 1e-5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm_weight = nn.Parameter(torch.ones(audio_encoder_hidden_size))
|
||||
self.norm_eps = norm_eps
|
||||
self.fc1 = nn.Linear(audio_encoder_hidden_size, intermediate_size, bias=False)
|
||||
self.fc2 = nn.Linear(intermediate_size, text_hidden_size, bias=False)
|
||||
self.activation = activation
|
||||
|
||||
def _rms_norm(self, x: torch.Tensor) -> torch.Tensor:
|
||||
dtype = x.dtype
|
||||
x = x.float()
|
||||
variance = x.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + self.norm_eps)
|
||||
return (self.norm_weight.float() * x).to(dtype)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
hidden_states = self._rms_norm(hidden_states)
|
||||
hidden_states = self.fc1(hidden_states)
|
||||
if self.activation == "relu2":
|
||||
hidden_states = F.relu(hidden_states).pow(2)
|
||||
elif self.activation == "gelu":
|
||||
hidden_states = F.gelu(hidden_states)
|
||||
else:
|
||||
raise ValueError(f"Unsupported audio projector activation: {self.activation}")
|
||||
return self.fc2(hidden_states)
|
||||
|
||||
|
||||
def _build_qwen2_audio_encoder(audio_config: dict) -> nn.Module:
|
||||
from transformers.models.qwen2_audio.configuration_qwen2_audio import (
|
||||
Qwen2AudioEncoderConfig,
|
||||
)
|
||||
from transformers.models.qwen2_audio.modeling_qwen2_audio import Qwen2AudioEncoder
|
||||
|
||||
cfg = Qwen2AudioEncoderConfig(**dict(audio_config))
|
||||
return Qwen2AudioEncoder(cfg)
|
||||
|
||||
|
||||
class AudexAudioFeatureInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- c: total number of 30s clips (flattened across audio items)
|
||||
- b: number of audio items
|
||||
"""
|
||||
|
||||
type: Literal["audio_features"] = "audio_features"
|
||||
input_features: Annotated[torch.Tensor, TensorShape("c", 128, 3000)]
|
||||
audio_num_clips: Annotated[list[int], None] = None
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
AudexMultiModalProcessor,
|
||||
info=AudexProcessingInfo,
|
||||
dummy_inputs=AudexDummyInputsBuilder,
|
||||
)
|
||||
class NemotronDenseAudexForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
):
|
||||
_LM_ARCHITECTURE = "NemotronDenseForCausalLM"
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("audio"):
|
||||
return SOUND_TOKEN
|
||||
raise ValueError(f"Only audio modality is supported, got {modality}")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
architectures=[self._LM_ARCHITECTURE],
|
||||
)
|
||||
llm_dtype = self.language_model.config.dtype
|
||||
assert isinstance(llm_dtype, torch.dtype)
|
||||
self.llm_dtype = llm_dtype
|
||||
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio_tower = _build_qwen2_audio_encoder(config.audio_config).to(llm_dtype)
|
||||
self.audio_projector = AudexProjector(
|
||||
audio_encoder_hidden_size=config.audio_encoder_hidden_size,
|
||||
intermediate_size=config.audio_projector_intermediate_size,
|
||||
text_hidden_size=config.hidden_size,
|
||||
activation=config.audio_projector_activation,
|
||||
norm_eps=config.audio_projector_norm_eps,
|
||||
).to(llm_dtype)
|
||||
|
||||
self.make_empty_intermediate_tensors = self.language_model.make_empty_intermediate_tensors
|
||||
|
||||
def get_language_model(self) -> nn.Module:
|
||||
return self.language_model
|
||||
|
||||
def _parse_and_validate_audio_input(self, **kwargs: object):
|
||||
input_features = kwargs.pop("input_features", None)
|
||||
audio_num_clips = kwargs.pop("audio_num_clips", None)
|
||||
if input_features is None:
|
||||
return None
|
||||
return AudexAudioFeatureInputs(
|
||||
type="audio_features",
|
||||
input_features=input_features,
|
||||
audio_num_clips=audio_num_clips,
|
||||
validate=False,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _encode_clips(self, input_features: torch.Tensor) -> torch.Tensor:
|
||||
"""(C, 128, 3000) -> (C, 750, hidden) via microbatched encoder+projector."""
|
||||
param = next(self.audio_tower.parameters())
|
||||
input_features = input_features.to(device=param.device, dtype=param.dtype)
|
||||
out_chunks = []
|
||||
for start in range(0, input_features.shape[0], AUDIO_ENCODER_MICRO_BATCH):
|
||||
chunk = input_features[start : start + AUDIO_ENCODER_MICRO_BATCH]
|
||||
hidden = self.audio_tower(input_features=chunk, return_dict=True).last_hidden_state
|
||||
out_chunks.append(self.audio_projector(hidden))
|
||||
return torch.cat(out_chunks, dim=0)
|
||||
|
||||
def _process_audio_input(self, audio_input) -> tuple[torch.Tensor, ...]:
|
||||
input_features = audio_input["input_features"]
|
||||
audio_num_clips = audio_input["audio_num_clips"]
|
||||
if input_features.ndim == 4:
|
||||
input_features = input_features.flatten(0, 1)
|
||||
|
||||
projected = self._encode_clips(input_features)
|
||||
per_clip = projected.reshape(projected.shape[0], -1, projected.shape[-1])
|
||||
|
||||
if audio_num_clips is None:
|
||||
num_clips_list = [projected.shape[0]]
|
||||
elif isinstance(audio_num_clips, torch.Tensor):
|
||||
num_clips_list = audio_num_clips.flatten().tolist()
|
||||
else:
|
||||
num_clips_list = [int(x) for x in audio_num_clips]
|
||||
|
||||
grouped: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for n in num_clips_list:
|
||||
clips = per_clip[offset : offset + n]
|
||||
grouped.append(clips.reshape(-1, clips.shape[-1]))
|
||||
offset += n
|
||||
return tuple(grouped)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
if audio_input is None:
|
||||
return []
|
||||
return self._process_audio_input(audio_input)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
return self.language_model.model(input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata=None) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Split the combined weight stream into LLM / audio_tower / projector.
|
||||
|
||||
vLLM streams *all* ``.safetensors`` in the model dir; route
|
||||
``audio_encoder.*``/``audio_projector.*`` to the audio modules and
|
||||
leave the remaining ``model.*``/``lm_head.*`` names untouched so the
|
||||
wrapped NemotronDenseForCausalLM loader applies its own qkv fusion
|
||||
and remapping.
|
||||
"""
|
||||
loaded: set[str] = set()
|
||||
lm_weights: list[tuple[str, torch.Tensor]] = []
|
||||
enc_sd: dict[str, torch.Tensor] = {}
|
||||
proj_map: dict[str, torch.Tensor] = {}
|
||||
|
||||
for name, w in weights:
|
||||
if name.startswith("audio_encoder."):
|
||||
enc_sd[name[len("audio_encoder.") :]] = w
|
||||
elif name == "audio_projector.norm.weight":
|
||||
proj_map["norm_weight"] = w
|
||||
elif name.startswith("audio_projector."):
|
||||
proj_map[name[len("audio_projector.") :]] = w
|
||||
else:
|
||||
lm_weights.append((name, w))
|
||||
|
||||
lm_loaded = self.language_model.load_weights(lm_weights)
|
||||
loaded |= {f"language_model.{n}" for n in lm_loaded}
|
||||
|
||||
if not enc_sd:
|
||||
raise ValueError("No audio_encoder.* weights found in checkpoint stream.")
|
||||
missing, unexpected = self.audio_tower.load_state_dict(enc_sd, strict=False)
|
||||
if missing:
|
||||
raise ValueError(f"Missing audio_tower params: {list(missing)[:8]}")
|
||||
if unexpected:
|
||||
raise ValueError(f"Unexpected audio_tower keys: {list(unexpected)[:8]}")
|
||||
loaded |= {f"audio_tower.{k}" for k in enc_sd}
|
||||
|
||||
if set(proj_map) != {"norm_weight", "fc1.weight", "fc2.weight"}:
|
||||
raise ValueError(f"Unexpected audio_projector keys: {sorted(proj_map)}")
|
||||
self.audio_projector.load_state_dict(proj_map, strict=True)
|
||||
loaded |= {f"audio_projector.{k}" for k in proj_map}
|
||||
|
||||
return loaded
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
AudexMultiModalProcessor,
|
||||
info=AudexProcessingInfo,
|
||||
dummy_inputs=AudexDummyInputsBuilder,
|
||||
)
|
||||
class NemotronHAudexForConditionalGeneration(
|
||||
NemotronDenseAudexForConditionalGeneration,
|
||||
IsHybrid,
|
||||
HasInnerState,
|
||||
SupportsMambaPrefixCaching,
|
||||
):
|
||||
"""Audex 30B-A3B audio-understanding model (``checkpoint_folder_full``).
|
||||
|
||||
Mirrors nvidia/Nemotron-Labs-Audex-30B-A3B's
|
||||
``inference_scripts_vllm/audioqa_scripts/audex_30b_a3b_vllm/modeling_audex_vllm.py``:
|
||||
the same encoder/projector/processing as the dense 2B wrapper around
|
||||
vLLM's native hybrid Mamba+MoE ``NemotronHForCausalLM``, whose state
|
||||
hooks are delegated below.
|
||||
"""
|
||||
|
||||
_LM_ARCHITECTURE = "NemotronHForCausalLM"
|
||||
|
||||
is_hybrid = True
|
||||
has_inner_state = True
|
||||
supports_mamba_prefix_caching = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(cls, vllm_config):
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
|
||||
return NemotronHForCausalLM.get_mamba_state_shape_from_config(vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(cls, vllm_config):
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
|
||||
return NemotronHForCausalLM.get_mamba_state_dtype_from_config(vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls):
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
|
||||
return NemotronHForCausalLM.get_mamba_state_copy_func()
|
||||
@@ -0,0 +1,260 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
|
||||
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/audio_features.py
|
||||
# inference_scripts_vllm/audioqa_scripts/audex_2b_vllm/processing_audex_vllm.py
|
||||
"""Multimodal processing for Audex audio understanding (S2T / audio QA).
|
||||
|
||||
No unified HF ``ProcessorMixin`` exists for Audex (only a
|
||||
``WhisperFeatureExtractor`` under ``audio_preprocessor/``), so
|
||||
``_call_hf_processor`` is fully overridden: it tokenizes the prompt and
|
||||
builds NV-Whisper features with 30 s non-overlapping windows (last clip
|
||||
zero-padded), exactly mirroring the release HF path.
|
||||
|
||||
Placeholder expansion uses vLLM's ``PromptReplacement``: a single
|
||||
``<so_embedding>`` per audio expands to ``<so_start>`` + N*``<so_embedding>``
|
||||
+ ``<so_end>`` where N = num_clips * 750.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
|
||||
SOUND_TOKEN = "<so_embedding>"
|
||||
SOUND_START_TOKEN = "<so_start>"
|
||||
SOUND_END_TOKEN = "<so_end>"
|
||||
|
||||
SOUND_EMBEDDING_SIZE = 750
|
||||
SOUND_CLIP_DURATION = 30.0
|
||||
SOUND_TARGET_RATE = 16000
|
||||
|
||||
MAX_AUDIO_SECONDS = 900
|
||||
MAX_AUDIO_CLIPS = 30
|
||||
MAX_AUDIO_TOKENS = MAX_AUDIO_CLIPS * SOUND_EMBEDDING_SIZE
|
||||
|
||||
AUDIO_ENCODER_MICRO_BATCH = 8
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- features
|
||||
|
||||
|
||||
def normalize_audio(audio: np.ndarray) -> np.ndarray:
|
||||
"""Return mono float32 audio in [-1, 1], matching the Megatron/HF path."""
|
||||
audio = np.asarray(audio)
|
||||
if audio.ndim == 2:
|
||||
if audio.shape[1] <= 2:
|
||||
audio = audio.mean(axis=1)
|
||||
elif audio.shape[0] <= 2:
|
||||
audio = audio.mean(axis=0)
|
||||
else:
|
||||
raise ValueError(f"Unsupported audio shape: {audio.shape}")
|
||||
|
||||
if audio.dtype == np.int16:
|
||||
audio = audio.astype(np.float32) / 32768.0
|
||||
elif audio.dtype != np.float32:
|
||||
audio = audio.astype(np.float32)
|
||||
|
||||
max_abs = float(np.abs(audio).max()) if audio.size else 0.0
|
||||
if max_abs > 1.0:
|
||||
audio = audio / max_abs
|
||||
return audio.astype(np.float32, copy=False)
|
||||
|
||||
|
||||
def split_audio_into_clips(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = SOUND_TARGET_RATE,
|
||||
clip_duration: float = SOUND_CLIP_DURATION,
|
||||
) -> list[np.ndarray]:
|
||||
"""Split audio into fixed clips; pad the final clip to full length."""
|
||||
audio = normalize_audio(audio)
|
||||
clip_samples = int(round(sample_rate * clip_duration))
|
||||
if clip_samples <= 0:
|
||||
raise ValueError(f"Invalid clip_samples: {clip_samples}")
|
||||
if audio.size == 0:
|
||||
audio = np.zeros(1, dtype=np.float32)
|
||||
|
||||
num_clips = max(1, math.ceil(audio.shape[0] / clip_samples))
|
||||
clips: list[np.ndarray] = []
|
||||
for idx in range(num_clips):
|
||||
start = idx * clip_samples
|
||||
clip = audio[start : start + clip_samples]
|
||||
if clip.shape[0] < clip_samples:
|
||||
clip = np.pad(clip, (0, clip_samples - clip.shape[0]))
|
||||
clips.append(clip.astype(np.float32, copy=False))
|
||||
return clips
|
||||
|
||||
|
||||
def extract_whisper_features(
|
||||
feature_extractor,
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = SOUND_TARGET_RATE,
|
||||
clip_duration: float = SOUND_CLIP_DURATION,
|
||||
) -> torch.Tensor:
|
||||
"""Return NV-Whisper input features shaped (num_clips, 128, 3000)."""
|
||||
clips = split_audio_into_clips(audio, sample_rate=sample_rate, clip_duration=clip_duration)
|
||||
features = feature_extractor(
|
||||
clips,
|
||||
sampling_rate=sample_rate,
|
||||
return_tensors="pt",
|
||||
padding="max_length",
|
||||
return_attention_mask=False,
|
||||
)
|
||||
input_features = features.input_features
|
||||
if input_features.ndim != 3:
|
||||
raise ValueError(f"Expected 3D Whisper features, got {tuple(input_features.shape)}")
|
||||
return input_features
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- processor
|
||||
|
||||
|
||||
class AudexProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.model_config.hf_config
|
||||
|
||||
def get_feature_extractor(self, **kwargs: object):
|
||||
from transformers import AutoFeatureExtractor
|
||||
|
||||
config = self.get_hf_config()
|
||||
path = getattr(config, "audio_preprocessor_path", None) or "audio_preprocessor"
|
||||
candidate = Path(path)
|
||||
if not candidate.is_absolute():
|
||||
candidate = Path(self.ctx.model_config.model) / candidate
|
||||
return AutoFeatureExtractor.from_pretrained(str(candidate))
|
||||
|
||||
def get_data_parser(self):
|
||||
return MultiModalDataParser(target_sr=SOUND_TARGET_RATE, target_channels=1)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": 1}
|
||||
|
||||
def get_mm_max_tokens_per_item(self, seq_len: int, mm_counts: Mapping[str, int] | None = None) -> Mapping[str, int]:
|
||||
return {"audio": MAX_AUDIO_TOKENS}
|
||||
|
||||
def sound_token_ids(self) -> tuple[int, int, int]:
|
||||
cfg = self.get_hf_config()
|
||||
return (
|
||||
int(cfg.sound_start_token_id),
|
||||
int(cfg.sound_token_id),
|
||||
int(cfg.sound_end_token_id),
|
||||
)
|
||||
|
||||
|
||||
class AudexDummyInputsBuilder(BaseDummyInputsBuilder[AudexProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return SOUND_TOKEN * mm_counts.get("audio", 0)
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
audio_len = int(SOUND_CLIP_DURATION * SOUND_TARGET_RATE)
|
||||
overrides = mm_options.get("audio") if mm_options else None
|
||||
return {"audio": self._get_dummy_audios(length=audio_len, num_audios=num_audios, overrides=overrides)}
|
||||
|
||||
|
||||
class AudexMultiModalProcessor(BaseMultiModalProcessor[AudexProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_ids = tokenizer(prompt, add_special_tokens=False).input_ids
|
||||
|
||||
audios = mm_data.get("audios", []) or mm_data.get("audio", [])
|
||||
if not audios:
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
all_features: list[torch.Tensor] = []
|
||||
audio_num_clips: list[int] = []
|
||||
for item in audios:
|
||||
audio = item[0] if isinstance(item, tuple) else item
|
||||
audio = normalize_audio(np.asarray(audio))
|
||||
duration = len(audio) / SOUND_TARGET_RATE
|
||||
if duration > MAX_AUDIO_SECONDS:
|
||||
raise ValueError(f"Audio duration {duration:.1f}s exceeds MAX_AUDIO_SECONDS={MAX_AUDIO_SECONDS}s.")
|
||||
clips = split_audio_into_clips(audio, sample_rate=SOUND_TARGET_RATE, clip_duration=SOUND_CLIP_DURATION)
|
||||
if len(clips) > MAX_AUDIO_CLIPS:
|
||||
raise ValueError(f"Audio needs {len(clips)} clips > MAX_AUDIO_CLIPS={MAX_AUDIO_CLIPS}.")
|
||||
feats = extract_whisper_features(
|
||||
feature_extractor,
|
||||
audio,
|
||||
sample_rate=SOUND_TARGET_RATE,
|
||||
clip_duration=SOUND_CLIP_DURATION,
|
||||
)
|
||||
all_features.append(feats)
|
||||
audio_num_clips.append(feats.shape[0])
|
||||
|
||||
input_features = torch.cat(all_features, dim=0)
|
||||
return BatchFeature(
|
||||
dict(
|
||||
input_ids=[prompt_ids],
|
||||
input_features=input_features,
|
||||
audio_num_clips=torch.tensor(audio_num_clips, dtype=torch.long),
|
||||
),
|
||||
tensor_type="pt",
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
audio_num_clips = hf_inputs.get("audio_num_clips", torch.zeros(0, dtype=torch.long))
|
||||
return dict(
|
||||
input_features=MultiModalFieldConfig.flat_from_sizes("audio", audio_num_clips),
|
||||
audio_num_clips=MultiModalFieldConfig.batched("audio"),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, Any],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
so_start, so_embed, so_end = self.info.sound_token_ids()
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
num_clips = out_mm_data.get("audio_num_clips", torch.zeros(0, dtype=torch.long))
|
||||
num_clips_list = num_clips.flatten().tolist() if hasattr(num_clips, "flatten") else list(num_clips)
|
||||
|
||||
def get_replacement_audex(item_idx: int):
|
||||
n = int(num_clips_list[item_idx]) * SOUND_EMBEDDING_SIZE
|
||||
if n < 1:
|
||||
raise ValueError(f"Audio item {item_idx} produced 0 embeddings.")
|
||||
full = [so_start] + [so_embed] * n + [so_end]
|
||||
return PromptUpdateDetails.select_token_id(full, so_embed)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=[so_embed],
|
||||
replacement=get_replacement_audex,
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,414 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from the NemotronDense vLLM plugin bundled with
|
||||
# nvidia/Nemotron-Labs-Audex-2B (nemotron_dense_vllm_plugin), which is not
|
||||
# installed as a package; the model code is carried here instead.
|
||||
"""Audex thinker: the Nemotron dense decoder-only LM that autoregressively
|
||||
emits ``<speechcodec_N>`` tokens for TTS.
|
||||
|
||||
The thinker is a plain token-in/token-out vLLM model. Speech-codec token
|
||||
extraction happens in the stage input processors
|
||||
(``vllm_omni/model_executor/stage_input_processors/audex.py``), which read the
|
||||
sampled token stream directly, so no omni-specific output plumbing is needed
|
||||
here.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class NemotronDenseRMSNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
return self.weight * hidden_states.to(input_dtype)
|
||||
|
||||
|
||||
class NemotronDenseMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.up_proj = ColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
output_size=intermediate_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = get_act_fn(hidden_act)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
up, _ = self.up_proj(x)
|
||||
x = self.act_fn(up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class NemotronDenseAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Any,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
max_position_embeddings: int = 8192,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = getattr(config, "head_dim", None)
|
||||
if self.head_dim is None:
|
||||
self.head_dim = self.hidden_size // self.total_num_heads
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.total_num_heads,
|
||||
total_num_kv_heads=self.total_num_kv_heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
input_size=self.total_num_heads * self.head_dim,
|
||||
output_size=hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class NemotronDenseDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Any,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
attention_bias = getattr(config, "attention_bias", False) or getattr(config, "bias", False)
|
||||
self.self_attn = NemotronDenseAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=getattr(config, "num_key_value_heads", config.num_attention_heads),
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
quant_config=quant_config,
|
||||
bias=attention_bias,
|
||||
cache_config=cache_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
self.mlp = NemotronDenseMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
|
||||
self.post_attention_layernorm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class NemotronDenseModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: NemotronDenseDecoderLayer(
|
||||
config=config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = NemotronDenseRMSNorm(config.hidden_size, eps=config.norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
|
||||
param = params_dict[scale_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(scale_name)
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
|
||||
break
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class NemotronDenseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
],
|
||||
}
|
||||
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
assert hasattr(config, "norm_eps") and hasattr(config, "num_hidden_layers")
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.model = NemotronDenseModel(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head.weight = self.model.embed_tokens.weight
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size, scale=logit_scale)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
# The omni AR runner forwards extra bookkeeping kwargs (e.g.
|
||||
# sampling_metadata); this plain LM ignores them.
|
||||
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata: Any = None,
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
@@ -0,0 +1,178 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex TTA stage 1: XCodec1 RVQ codes → waveform.
|
||||
|
||||
Unlike the streaming causal speech decoder used for TTS, XCodec1 is a CNN
|
||||
codec that the official flow decodes over the full sequence, so this stage
|
||||
runs sync full-payload only: each request delivers its complete
|
||||
de-interleaved ``[frames, 4]`` codec-id payload once at finish and the
|
||||
decode happens in a single call.
|
||||
|
||||
The checkpoint (``hf-audio/xcodec-hubert-general-balanced``) is external to
|
||||
the Audex repo and loads through transformers remote code; the model
|
||||
skeleton is built from config at startup for vLLM's memory profiler and the
|
||||
weights arrive via the standard ``load_weights`` iteration over the
|
||||
snapshot's safetensors.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.audex.tta import (
|
||||
XCODEC1_CODEBOOK_SIZE,
|
||||
XCODEC1_NUM_CODEBOOKS,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Official cap on decoded frames (~10 s of audio at the codec frame rate).
|
||||
_DEFAULT_MAX_TTA_FRAMES = 500
|
||||
|
||||
|
||||
def _meta_str(value: Any) -> str | None:
|
||||
if isinstance(value, list | tuple):
|
||||
return _meta_str(value[0]) if value else None
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _codes_from_runtime_info(runtime_info: dict[str, Any] | None) -> torch.Tensor | None:
|
||||
"""Extract the ``[frames, 4]`` codec-id payload from the connector info."""
|
||||
if not isinstance(runtime_info, dict):
|
||||
return None
|
||||
codes = runtime_info.get("codes")
|
||||
if not isinstance(codes, dict):
|
||||
return None
|
||||
audio = codes.get("audio")
|
||||
if isinstance(audio, torch.Tensor):
|
||||
return audio
|
||||
if isinstance(audio, list | tuple) and audio:
|
||||
return torch.as_tensor(audio, dtype=torch.long)
|
||||
return None
|
||||
|
||||
|
||||
class AudexXCodec1(nn.Module):
|
||||
"""Stage-1 model for Audex TTA (GenerationModelRunner)."""
|
||||
|
||||
input_modalities = "audio"
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
from transformers import AutoConfig, AutoModel
|
||||
|
||||
config = AutoConfig.from_pretrained(self.model_path, trust_remote_code=True)
|
||||
self.codec = AutoModel.from_config(config, trust_remote_code=True)
|
||||
self.codec.eval()
|
||||
self._sample_rate = int(getattr(config, "sampling_rate", 16000))
|
||||
self._max_frames = int(getattr(config, "max_tta_frames", _DEFAULT_MAX_TTA_FRAMES))
|
||||
|
||||
# -------------------- vLLM model interface --------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
# This stage ignores token embeddings; keep a stable dummy for the runner.
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
|
||||
return None
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
state_dict = dict(weights)
|
||||
self.codec.load_state_dict(state_dict, strict=True)
|
||||
device = self.vllm_config.device_config.device
|
||||
self.codec.to(device=device, dtype=torch.float32)
|
||||
return {f"codec.{name}" for name in state_dict}
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
# The runner may unpack the NamedTuple in transit; rebuild it.
|
||||
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
|
||||
return OmniOutput(*model_outputs)
|
||||
raise TypeError(f"AudexXCodec1 expected OmniOutput, got {type(model_outputs)}")
|
||||
|
||||
# -------------------- decoding --------------------
|
||||
|
||||
def _decode_codes(self, req_id: str | None, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""Decode a ``[frames, 4]`` codec-id payload to a mono waveform."""
|
||||
if codes.numel() == 0:
|
||||
# Zero codec tokens for the whole request. Do NOT raise: an
|
||||
# exception in forward() kills the engine core for every request.
|
||||
# Returning empty audio lets the serving layer fail this request.
|
||||
logger.error("Audex TTA: request %s finished with no codec tokens; no audio generated", req_id)
|
||||
return torch.empty(0, dtype=torch.float32)
|
||||
if codes.ndim != 2 or codes.shape[1] != XCODEC1_NUM_CODEBOOKS:
|
||||
logger.error(
|
||||
"Audex TTA: request %s payload has shape %s, expected [frames, %d]; no audio generated",
|
||||
req_id,
|
||||
tuple(codes.shape),
|
||||
XCODEC1_NUM_CODEBOOKS,
|
||||
)
|
||||
return torch.empty(0, dtype=torch.float32)
|
||||
|
||||
device = self.vllm_config.device_config.device
|
||||
with torch.inference_mode():
|
||||
frames = codes[: self._max_frames].to(device=device, dtype=torch.long)
|
||||
# [frames, 4] codec ids -> [1, 4, frames] per-layer indices.
|
||||
layer_offsets = (
|
||||
torch.arange(XCODEC1_NUM_CODEBOOKS, device=frames.device, dtype=frames.dtype) * XCODEC1_CODEBOOK_SIZE
|
||||
)
|
||||
per_layer = (frames - layer_offsets.view(1, -1)).clamp(0, XCODEC1_CODEBOOK_SIZE - 1)
|
||||
stacked = per_layer.transpose(0, 1).unsqueeze(0)
|
||||
decoded = self.codec.decode(stacked)
|
||||
audio = decoded.audio_values if hasattr(decoded, "audio_values") else decoded[0]
|
||||
return audio.reshape(-1).to(dtype=torch.float32, device="cpu")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
runtime_infos = runtime_additional_information or []
|
||||
seq_token_counts = kwargs.get("seq_token_counts")
|
||||
if seq_token_counts is not None:
|
||||
num_reqs = len(seq_token_counts)
|
||||
elif input_ids is not None and input_ids.numel() > 0 and not runtime_infos:
|
||||
num_reqs = 1
|
||||
else:
|
||||
num_reqs = len(runtime_infos)
|
||||
|
||||
audios: list[torch.Tensor] = []
|
||||
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
|
||||
for i in range(num_reqs):
|
||||
info = runtime_infos[i] if i < len(runtime_infos) else {}
|
||||
info = info if isinstance(info, dict) else {}
|
||||
meta = info.get("meta", {})
|
||||
meta = meta if isinstance(meta, dict) else {}
|
||||
req_id = _meta_str(meta.get("req_id"))
|
||||
codes = _codes_from_runtime_info(info)
|
||||
if codes is None:
|
||||
audios.append(torch.empty(0, dtype=torch.float32))
|
||||
continue
|
||||
audios.append(self._decode_codes(req_id, codes))
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audios, "sr": [sr_tensor] * num_reqs},
|
||||
)
|
||||
@@ -0,0 +1,567 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
|
||||
# inference_scripts_vllm/audiogen_scripts/cfg_logits_processor.py
|
||||
# inference_scripts_vllm/audiogen_scripts/vllm_cfg_patch.py
|
||||
# Divergences from the official files are commented at the divergence site.
|
||||
"""Classifier-free guidance for the Audex thinker stage.
|
||||
|
||||
CFG pairs a conditional request with an unconditional (null-prompt)
|
||||
companion in the same engine and blends their logits every step:
|
||||
|
||||
blended = uncond + cfg_scale * (cond - uncond)
|
||||
|
||||
Both rows receive the blended logits and the sampled token is copied from
|
||||
the cond row to the uncond row, so the two sequences stay token-identical.
|
||||
Requests opt in via ``SamplingParams.extra_args``:
|
||||
|
||||
cond: {"cfg_scale": 1.5, "cfg_role": "cond", "cfg_pair_id": <id>}
|
||||
uncond: {"cfg_scale": 1.5, "cfg_role": "uncond", "cfg_pair_id": <id>}
|
||||
|
||||
Blending is only correct when both pair members decode the same position in
|
||||
the same engine step, so ``apply_cfg_patches()`` makes the vLLM scheduler
|
||||
pair-aware: a lone member waits for its partner, partners stay adjacent,
|
||||
their ``num_computed_tokens`` are equalized after every schedule, and they
|
||||
finish together. The patch is a no-op for engines that never see a CFG
|
||||
request (the pair registry stays empty).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_COND = "cond"
|
||||
_UNCOND = "uncond"
|
||||
|
||||
|
||||
def _cfg_extra(request: Any, key: str) -> Any:
|
||||
sampling_params = getattr(request, "sampling_params", None)
|
||||
extra_args = getattr(sampling_params, "extra_args", None)
|
||||
if extra_args:
|
||||
return extra_args.get(key)
|
||||
return None
|
||||
|
||||
|
||||
class AudexCFGLogitsProcessor(LogitsProcessor):
|
||||
"""Blend cond/uncond logits for classifier-free guidance.
|
||||
|
||||
Pairs are matched by ``cfg_pair_id``. For each pair the blended logits
|
||||
are written to *both* rows so the sampler picks the same token; the
|
||||
post-sampling copy of the cond token into the uncond slot is installed
|
||||
by :meth:`_ensure_sample_patched` (which must run inside each worker
|
||||
process, hence from ``__init__`` rather than a main-process patch).
|
||||
"""
|
||||
|
||||
_sample_patched = False
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams) -> None:
|
||||
extra_args = params.extra_args
|
||||
if not extra_args:
|
||||
return
|
||||
role = extra_args.get("cfg_role")
|
||||
if role is not None and role not in (_COND, _UNCOND):
|
||||
raise ValueError(f"cfg_role must be 'cond' or 'uncond', got '{role}'")
|
||||
scale = extra_args.get("cfg_scale")
|
||||
if scale is not None and (not isinstance(scale, int | float) or scale < 1.0):
|
||||
raise ValueError(f"cfg_scale must be >= 1.0, got {scale}")
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool) -> None:
|
||||
self._info: dict[int, dict[str, Any]] = {}
|
||||
self._output_tokens: dict[int, list[int]] = {}
|
||||
self._pairs: list[tuple[int, int, float]] = []
|
||||
self._dirty = True
|
||||
self._ensure_sample_patched()
|
||||
|
||||
@classmethod
|
||||
def _ensure_sample_patched(cls) -> None:
|
||||
"""Install the post-sampling cond→uncond token copy (once per process)."""
|
||||
if cls._sample_patched:
|
||||
return
|
||||
cls._sample_patched = True
|
||||
|
||||
# Divergence from the official file: it patches vLLM's
|
||||
# ``GPUModelRunner._sample``, but vllm-omni's AR runner overrides
|
||||
# ``_sample``, so the base-class patch would never run for the Audex
|
||||
# thinker stage. Patch the omni AR runner instead.
|
||||
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
|
||||
|
||||
orig_sample = GPUARModelRunner._sample
|
||||
|
||||
def _sample_with_cfg_sync(self, logits, spec_decode_metadata):
|
||||
sampler_output = orig_sample(self, logits, spec_decode_metadata)
|
||||
for proc in self.input_batch.logitsprocs.all:
|
||||
if isinstance(proc, AudexCFGLogitsProcessor) and proc._pairs:
|
||||
sampled = sampler_output.sampled_token_ids
|
||||
num_rows = sampled.shape[0] if hasattr(sampled, "shape") else len(sampled)
|
||||
for cond_idx, uncond_idx, _ in proc._pairs:
|
||||
# Steps that sample fewer rows than the persistent
|
||||
# batch (pure-prefill / partially scheduled steps)
|
||||
# have nothing to sync for this pair; the scheduler
|
||||
# pair patches keep both members step-locked on the
|
||||
# decode steps that matter.
|
||||
if cond_idx < num_rows and uncond_idx < num_rows:
|
||||
sampled[uncond_idx] = sampled[cond_idx]
|
||||
break
|
||||
return sampler_output
|
||||
|
||||
_sample_with_cfg_sync._audex_cfg_sync = True # type: ignore[attr-defined]
|
||||
GPUARModelRunner._sample = _sample_with_cfg_sync
|
||||
logger.info("AudexCFGLogitsProcessor: patched GPUARModelRunner._sample for CFG token sync")
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
return False
|
||||
|
||||
def _reset(self) -> None:
|
||||
self._info.clear()
|
||||
self._output_tokens.clear()
|
||||
self._pairs.clear()
|
||||
self._dirty = True
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None) -> None:
|
||||
if batch_update is None:
|
||||
return
|
||||
|
||||
for idx in batch_update.removed:
|
||||
self._info.pop(idx, None)
|
||||
self._output_tokens.pop(idx, None)
|
||||
|
||||
if not self._info and batch_update.added:
|
||||
self._reset()
|
||||
|
||||
for idx, params, _, output_token_ids in batch_update.added:
|
||||
extra_args = params.extra_args if params else None
|
||||
if extra_args and extra_args.get("cfg_role") in (_COND, _UNCOND):
|
||||
self._info[idx] = {
|
||||
"role": extra_args["cfg_role"],
|
||||
"cfg_scale": float(extra_args.get("cfg_scale", 1.0)),
|
||||
"pair_id": extra_args.get("cfg_pair_id"),
|
||||
}
|
||||
self._output_tokens[idx] = output_token_ids
|
||||
else:
|
||||
self._info.pop(idx, None)
|
||||
self._output_tokens.pop(idx, None)
|
||||
self._dirty = True
|
||||
|
||||
if self._info:
|
||||
for src, dst, direction in batch_update.moved:
|
||||
src_info = self._info.pop(src, None)
|
||||
dst_info = self._info.pop(dst, None)
|
||||
src_tokens = self._output_tokens.pop(src, None)
|
||||
dst_tokens = self._output_tokens.pop(dst, None)
|
||||
if src_info is not None:
|
||||
self._info[dst] = src_info
|
||||
if src_tokens is not None:
|
||||
self._output_tokens[dst] = src_tokens
|
||||
if direction == MoveDirectionality.SWAP:
|
||||
if dst_info is not None:
|
||||
self._info[src] = dst_info
|
||||
if dst_tokens is not None:
|
||||
self._output_tokens[src] = dst_tokens
|
||||
self._dirty = True
|
||||
|
||||
def _rebuild_pairs(self) -> None:
|
||||
by_pair: dict[str, dict[str, tuple[int, float]]] = {}
|
||||
for idx, info in self._info.items():
|
||||
pair_id = info.get("pair_id")
|
||||
if pair_id is None:
|
||||
continue
|
||||
by_pair.setdefault(pair_id, {})[info["role"]] = (idx, info["cfg_scale"])
|
||||
|
||||
self._pairs = [
|
||||
(roles[_COND][0], roles[_UNCOND][0], roles[_COND][1])
|
||||
for roles in by_pair.values()
|
||||
if _COND in roles and _UNCOND in roles
|
||||
]
|
||||
self._dirty = False
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if not self._info:
|
||||
return logits
|
||||
|
||||
if self._dirty:
|
||||
self._rebuild_pairs()
|
||||
|
||||
num_rows = logits.shape[0]
|
||||
for cond_idx, uncond_idx, cfg_scale in self._pairs:
|
||||
# Rows beyond this step's logits (pure-prefill / partially
|
||||
# scheduled steps) have nothing to blend; decode steps carry
|
||||
# both pair rows thanks to the scheduler pair patches.
|
||||
if cond_idx >= num_rows or uncond_idx >= num_rows:
|
||||
continue
|
||||
blended = logits[uncond_idx] + cfg_scale * (logits[cond_idx] - logits[uncond_idx])
|
||||
logits[cond_idx] = blended
|
||||
logits[uncond_idx] = blended
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler pair synchronization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _wait_queues(scheduler: Any) -> list[Any]:
|
||||
# Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks
|
||||
# structurally not-ready requests in a second ``skipped_waiting`` queue;
|
||||
# pair-hold must cover both queues or a lone member parked there would
|
||||
# be admitted without its partner.
|
||||
queues = [scheduler.waiting]
|
||||
skipped = getattr(scheduler, "skipped_waiting", None)
|
||||
if skipped is not None:
|
||||
queues.append(skipped)
|
||||
return queues
|
||||
|
||||
|
||||
def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None = None) -> bool:
|
||||
pair_id = scheduler._cfg_req_to_pair.get(request_id)
|
||||
if pair_id is None:
|
||||
return True
|
||||
roles = scheduler._cfg_pairs.get(pair_id, {})
|
||||
if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()):
|
||||
return False
|
||||
if blocked_ids:
|
||||
# A partner parked in skipped_waiting (e.g. waiting for remote KVs)
|
||||
# may fail promotion this step while this member gets scheduled from
|
||||
# waiting — hold this member until the partner is schedulable.
|
||||
for rid in roles.values():
|
||||
if rid != request_id and rid in blocked_ids:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Schedule steps a lone pair member may wait for its partner. Partners arrive
|
||||
# within a handful of steps in practice; exhausting this budget means the
|
||||
# companion was never created (e.g. prompt expansion failed), and the request
|
||||
# is released to decode unguided instead of hanging forever.
|
||||
_MAX_PAIR_HOLD_STEPS = 512
|
||||
|
||||
|
||||
def _drop_stale_pair(scheduler: Any, request_id: str) -> None:
|
||||
pair_id = scheduler._cfg_req_to_pair.get(request_id)
|
||||
if pair_id is None:
|
||||
return
|
||||
for rid in scheduler._cfg_pairs.pop(pair_id, {}).values():
|
||||
scheduler._cfg_req_to_pair.pop(rid, None)
|
||||
logger.warning(
|
||||
"CFG pair %s: partner of %s never arrived after %d schedule steps; releasing the request unguided",
|
||||
pair_id,
|
||||
request_id,
|
||||
_MAX_PAIR_HOLD_STEPS,
|
||||
)
|
||||
|
||||
|
||||
def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]:
|
||||
"""Pull CFG requests whose partner is not yet admitted out of the wait queues.
|
||||
|
||||
The engine schedules as soon as the first member of a pair arrives over
|
||||
IPC; without the hold, the lone member prefills one step early and the
|
||||
pair is permanently offset by one token. Held requests are re-prepended
|
||||
after the schedule step.
|
||||
"""
|
||||
hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {}
|
||||
scheduler._cfg_hold_counts = hold_counts
|
||||
|
||||
skipped = getattr(scheduler, "skipped_waiting", None)
|
||||
blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set()
|
||||
|
||||
held: list[tuple[Any, Any]] = []
|
||||
for queue in _wait_queues(scheduler):
|
||||
# A member is only "blocked" for its partner's sake when it sits in
|
||||
# the OTHER queue; members of the queue being scanned move together.
|
||||
partner_blockers = blocked_ids if queue is not skipped else set()
|
||||
held_here = []
|
||||
for request in list(queue):
|
||||
if _pair_complete(scheduler, request.request_id, partner_blockers):
|
||||
hold_counts.pop(request.request_id, None)
|
||||
continue
|
||||
count = hold_counts.get(request.request_id, 0) + 1
|
||||
if count > _MAX_PAIR_HOLD_STEPS:
|
||||
hold_counts.pop(request.request_id, None)
|
||||
_drop_stale_pair(scheduler, request.request_id)
|
||||
continue
|
||||
hold_counts[request.request_id] = count
|
||||
held_here.append(request)
|
||||
if held_here:
|
||||
queue.remove_requests(held_here)
|
||||
held.extend((queue, req) for req in held_here)
|
||||
return held
|
||||
|
||||
|
||||
def _release_held(held: list[tuple[Any, Any]]) -> None:
|
||||
for queue, request in reversed(held):
|
||||
queue.prepend_request(request)
|
||||
|
||||
|
||||
def _reorder_waiting_for_cfg(scheduler: Any) -> None:
|
||||
"""Move CFG pair partners adjacent in the FCFS waiting queue."""
|
||||
waiting = scheduler.waiting
|
||||
# Only the FCFS queue (a deque) has caller-controlled ordering; priority
|
||||
# queues order themselves and pair sync then relies on hold + equalize.
|
||||
if not isinstance(waiting, deque) or len(waiting) < 2:
|
||||
return
|
||||
|
||||
requests = list(waiting)
|
||||
waiting.clear()
|
||||
|
||||
seen: set[str] = set()
|
||||
result: list[Any] = []
|
||||
for request in requests:
|
||||
rid = request.request_id
|
||||
if rid in seen:
|
||||
continue
|
||||
seen.add(rid)
|
||||
result.append(request)
|
||||
|
||||
pair_id = scheduler._cfg_req_to_pair.get(rid)
|
||||
if pair_id is None:
|
||||
continue
|
||||
for partner_id in scheduler._cfg_pairs.get(pair_id, {}).values():
|
||||
if partner_id != rid and partner_id not in seen:
|
||||
for candidate in requests:
|
||||
if candidate.request_id == partner_id:
|
||||
seen.add(partner_id)
|
||||
result.append(candidate)
|
||||
break
|
||||
|
||||
waiting.extend(result)
|
||||
|
||||
|
||||
def _equalize_cfg_pair_progress(scheduler: Any, scheduler_output: Any) -> None:
|
||||
"""Bring both members of every CFG pair to the same ``num_computed_tokens``.
|
||||
|
||||
Prefix-cache hits or unequal chunked-prefill budget can leave one member
|
||||
ahead after a schedule step; reduce the faster member's allocation so
|
||||
both land on the same position. Over-allocated KV blocks remain and are
|
||||
consumed next step — nothing leaks.
|
||||
"""
|
||||
for roles in scheduler._cfg_pairs.values():
|
||||
cond_id = roles.get(_COND)
|
||||
uncond_id = roles.get(_UNCOND)
|
||||
if not cond_id or not uncond_id:
|
||||
continue
|
||||
|
||||
cond_sched = scheduler_output.num_scheduled_tokens.get(cond_id, 0)
|
||||
uncond_sched = scheduler_output.num_scheduled_tokens.get(uncond_id, 0)
|
||||
if cond_sched == 0 or uncond_sched == 0:
|
||||
continue
|
||||
|
||||
cond_req = scheduler.requests.get(cond_id)
|
||||
uncond_req = scheduler.requests.get(uncond_id)
|
||||
if not cond_req or not uncond_req:
|
||||
continue
|
||||
|
||||
if cond_req.num_computed_tokens == uncond_req.num_computed_tokens:
|
||||
continue
|
||||
|
||||
target = min(cond_req.num_computed_tokens, uncond_req.num_computed_tokens)
|
||||
|
||||
feasible = all(
|
||||
req.num_computed_tokens - target < sched
|
||||
for req, sched in ((cond_req, cond_sched), (uncond_req, uncond_sched))
|
||||
)
|
||||
if not feasible:
|
||||
continue
|
||||
|
||||
for req_id, req, orig_sched in (
|
||||
(cond_id, cond_req, cond_sched),
|
||||
(uncond_id, uncond_req, uncond_sched),
|
||||
):
|
||||
diff = req.num_computed_tokens - target
|
||||
if diff > 0:
|
||||
req.num_computed_tokens = target
|
||||
scheduler_output.num_scheduled_tokens[req_id] = orig_sched - diff
|
||||
scheduler_output.total_num_scheduled_tokens -= diff
|
||||
logger.debug(
|
||||
"CFG equalize %s: %d -> %d scheduled, computed -> %d",
|
||||
req_id,
|
||||
orig_sched,
|
||||
orig_sched - diff,
|
||||
target,
|
||||
)
|
||||
|
||||
|
||||
def _drop_split_pairs(scheduler: Any) -> None:
|
||||
"""Deregister a desynced CFG pair (one member preempted / left behind).
|
||||
|
||||
Manual re-preemption surgery is unsafe against the upstream scheduler.
|
||||
Instead, a pair that split anyway (KV-pressure preemption of exactly one
|
||||
member, or unrecoverable progress skew) is dropped from the registry:
|
||||
the cond request continues UNGUIDED (its logits are never blended again)
|
||||
rather than being re-paired against a desynced partner, and the
|
||||
companion's discarded output no longer matters.
|
||||
"""
|
||||
if not scheduler._cfg_pairs:
|
||||
return
|
||||
|
||||
running_ids = {req.request_id for req in scheduler.running}
|
||||
to_drop: list[tuple[str, str, str, str]] = []
|
||||
for pair_id, roles in scheduler._cfg_pairs.items():
|
||||
cond_id = roles.get(_COND)
|
||||
uncond_id = roles.get(_UNCOND)
|
||||
if cond_id is None or uncond_id is None:
|
||||
continue
|
||||
if cond_id not in scheduler.requests or uncond_id not in scheduler.requests:
|
||||
continue
|
||||
|
||||
cond_running = cond_id in running_ids
|
||||
uncond_running = uncond_id in running_ids
|
||||
if cond_running != uncond_running:
|
||||
# Split only counts once decoding started: a lone member that has
|
||||
# computed tokens while its partner was pushed back is desynced.
|
||||
runner = scheduler.requests.get(cond_id if cond_running else uncond_id)
|
||||
if runner is not None and runner.num_computed_tokens > 0:
|
||||
to_drop.append((pair_id, cond_id, uncond_id, "one member preempted"))
|
||||
continue
|
||||
|
||||
if cond_running and uncond_running:
|
||||
cond_req = scheduler.requests.get(cond_id)
|
||||
uncond_req = scheduler.requests.get(uncond_id)
|
||||
if cond_req and uncond_req and cond_req.num_computed_tokens != uncond_req.num_computed_tokens:
|
||||
logger.warning(
|
||||
"CFG pair progress mismatch (equalize will retry): %s@%d vs %s@%d",
|
||||
cond_id,
|
||||
cond_req.num_computed_tokens,
|
||||
uncond_id,
|
||||
uncond_req.num_computed_tokens,
|
||||
)
|
||||
|
||||
for pair_id, cond_id, uncond_id, reason in to_drop:
|
||||
scheduler._cfg_pairs.pop(pair_id, None)
|
||||
scheduler._cfg_req_to_pair.pop(cond_id, None)
|
||||
scheduler._cfg_req_to_pair.pop(uncond_id, None)
|
||||
logger.warning(
|
||||
"CFG pair %s split (%s); releasing cond=%s unguided, uncond=%s output is discarded",
|
||||
pair_id,
|
||||
reason,
|
||||
cond_id,
|
||||
uncond_id,
|
||||
)
|
||||
|
||||
|
||||
_patches_applied = False
|
||||
|
||||
|
||||
def cfg_patches_applied() -> bool:
|
||||
"""True once :func:`apply_cfg_patches` has patched the scheduler in this process."""
|
||||
return _patches_applied
|
||||
|
||||
|
||||
def _patch_scheduler() -> None:
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
orig_init = Scheduler.__init__
|
||||
orig_add = Scheduler.add_request
|
||||
orig_finish = Scheduler.finish_requests
|
||||
orig_schedule = Scheduler.schedule
|
||||
|
||||
def _init(self, *args: Any, **kwargs: Any) -> None:
|
||||
orig_init(self, *args, **kwargs)
|
||||
self._cfg_pairs: dict[str, dict[str, str]] = {}
|
||||
self._cfg_req_to_pair: dict[str, str] = {}
|
||||
|
||||
def _add(self, request: Any) -> None:
|
||||
orig_add(self, request)
|
||||
pair_id = _cfg_extra(request, "cfg_pair_id")
|
||||
role = _cfg_extra(request, "cfg_role")
|
||||
if pair_id and role:
|
||||
self._cfg_pairs.setdefault(pair_id, {})[role] = request.request_id
|
||||
self._cfg_req_to_pair[request.request_id] = pair_id
|
||||
# An asymmetric prefix-cache hit (the cond prompt shares cached
|
||||
# branches; the <unk> null prompt does not) can advance one
|
||||
# member past what post-schedule equalization can claw back.
|
||||
# Skip prefix-cache reads for CFG requests entirely.
|
||||
if hasattr(request, "skip_reading_prefix_cache"):
|
||||
request.skip_reading_prefix_cache = True
|
||||
|
||||
def _finish(self, request_ids: Any, finished_status: Any) -> Any:
|
||||
if request_ids is None:
|
||||
result = orig_finish(self, request_ids, finished_status)
|
||||
self._cfg_pairs.clear()
|
||||
self._cfg_req_to_pair.clear()
|
||||
return result
|
||||
|
||||
if isinstance(request_ids, str):
|
||||
request_ids = {request_ids}
|
||||
else:
|
||||
request_ids = set(request_ids)
|
||||
|
||||
# Finish both members together: a surviving partner would decode
|
||||
# unpaired garbage (its logits are never blended again).
|
||||
partner_ids: set[str] = set()
|
||||
for req_id in request_ids:
|
||||
pair_id = self._cfg_req_to_pair.get(req_id)
|
||||
if pair_id is None:
|
||||
continue
|
||||
for rid in self._cfg_pairs.get(pair_id, {}).values():
|
||||
if rid != req_id and rid in self.requests:
|
||||
partner_ids.add(rid)
|
||||
|
||||
all_ids = request_ids | partner_ids
|
||||
result = orig_finish(self, all_ids, finished_status)
|
||||
|
||||
for req_id in all_ids:
|
||||
pair_id = self._cfg_req_to_pair.pop(req_id, None)
|
||||
if pair_id:
|
||||
self._cfg_pairs.pop(pair_id, None)
|
||||
return result
|
||||
|
||||
def _schedule(self, *args: Any, **kwargs: Any) -> Any:
|
||||
if not self._cfg_pairs:
|
||||
return orig_schedule(self, *args, **kwargs)
|
||||
|
||||
_reorder_waiting_for_cfg(self)
|
||||
held = _hold_incomplete_pairs(self)
|
||||
|
||||
# Chunked prefill can split one pair member's prompt across steps
|
||||
# while the other fits in a single step; halving the long-prefill
|
||||
# threshold makes both chunk, and equalize aligns the remainder.
|
||||
scheduler_config = getattr(self, "scheduler_config", None)
|
||||
orig_threshold = getattr(scheduler_config, "long_prefill_token_threshold", None)
|
||||
if scheduler_config is not None and orig_threshold is not None:
|
||||
scheduler_config.long_prefill_token_threshold = self.max_num_scheduled_tokens // 2
|
||||
try:
|
||||
scheduler_output = orig_schedule(self, *args, **kwargs)
|
||||
finally:
|
||||
if scheduler_config is not None and orig_threshold is not None:
|
||||
scheduler_config.long_prefill_token_threshold = orig_threshold
|
||||
_release_held(held)
|
||||
|
||||
_equalize_cfg_pair_progress(self, scheduler_output)
|
||||
_drop_split_pairs(self)
|
||||
return scheduler_output
|
||||
|
||||
_schedule._audex_cfg_patched = True # type: ignore[attr-defined]
|
||||
|
||||
Scheduler.__init__ = _init
|
||||
Scheduler.add_request = _add
|
||||
Scheduler.finish_requests = _finish
|
||||
Scheduler.schedule = _schedule
|
||||
|
||||
|
||||
def apply_cfg_patches() -> None:
|
||||
"""Make the vLLM v1 scheduler CFG-pair-aware. Idempotent, per process.
|
||||
|
||||
Must run in the engine-core process before ``Scheduler`` is constructed
|
||||
(the ``__init__`` wrapper installs the pair registry).
|
||||
"""
|
||||
global _patches_applied
|
||||
if _patches_applied:
|
||||
return
|
||||
_patches_applied = True
|
||||
logger.info("Applying Audex CFG scheduler patches to vLLM v1")
|
||||
_patch_scheduler()
|
||||
@@ -0,0 +1,186 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex checkpoint layout preparation.
|
||||
|
||||
The nvidia/Nemotron-Labs-Audex-2B repo deduplicates weights: the
|
||||
``checkpoint_folder_audiogen`` (and ``checkpoint_folder_textonly``) index
|
||||
files reference safetensors shards that physically live only under
|
||||
``checkpoint_folder_full``. The official repo ships a prepare script that
|
||||
symlinks them; this replicates that step automatically so users can pass the
|
||||
repo root without any manual preparation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_WEIGHT_SOURCE_FOLDER = "checkpoint_folder_full"
|
||||
|
||||
# Download profiles: each deployment pulls only the repo subset its stages
|
||||
# need. The dedup weight shards are appended per-model at call time from
|
||||
# the audiogen index (see _dedup_shard_patterns); the static lists here
|
||||
# hold only the model-independent folders.
|
||||
_SNAPSHOT_PROFILE_PATTERNS = {
|
||||
"tts": [
|
||||
"config.json",
|
||||
"checkpoint_folder_audiogen/*",
|
||||
"audex_causal_speech_decoder/*",
|
||||
],
|
||||
# TTA reuses the audiogen thinker; waveform decode happens in the
|
||||
# external XCodec1 checkpoint (see ensure_xcodec1_snapshot), so the
|
||||
# bundled speech decoder is not pulled.
|
||||
"tta": [
|
||||
"config.json",
|
||||
"checkpoint_folder_audiogen/*",
|
||||
],
|
||||
# The audio-capable full checkpoint: both weight shards, audio
|
||||
# preprocessor assets, chat template, and remote-code modeling files.
|
||||
"full": [
|
||||
"config.json",
|
||||
f"{_WEIGHT_SOURCE_FOLDER}/*",
|
||||
],
|
||||
}
|
||||
|
||||
XCODEC1_DEFAULT_REPO = "hf-audio/xcodec-hubert-general-balanced"
|
||||
|
||||
_AUDIOGEN_INDEX_FILE = "checkpoint_folder_audiogen/model.safetensors.index.json"
|
||||
|
||||
|
||||
def _download_audiogen_index(model: str) -> str:
|
||||
"""Resolve the audiogen dedup index to a local path (tiny, cache-first).
|
||||
|
||||
Cache-first keeps warm-cache boots offline-safe and avoids a Hub
|
||||
metadata round trip per stage-engine boot; a cache miss falls back to
|
||||
the network.
|
||||
"""
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
try:
|
||||
return hf_hub_download(model, _AUDIOGEN_INDEX_FILE, local_files_only=True)
|
||||
except Exception:
|
||||
return hf_hub_download(model, _AUDIOGEN_INDEX_FILE)
|
||||
|
||||
|
||||
def _dedup_shard_patterns(model: str) -> list[str]:
|
||||
"""Shards under the full checkpoint referenced by the audiogen index.
|
||||
|
||||
The audiogen folder deduplicates its LM weights into
|
||||
``checkpoint_folder_full`` shards (1 of 2 on the 2B, a subset of 14 on
|
||||
the 30B-A3B); the ``tts``/``tta`` profiles must download exactly those.
|
||||
Falls back to every full-checkpoint shard when the index is unreadable.
|
||||
"""
|
||||
try:
|
||||
with open(_download_audiogen_index(model)) as f:
|
||||
weight_map = json.load(f).get("weight_map", {})
|
||||
shards = sorted(set(weight_map.values()))
|
||||
if shards:
|
||||
return [f"{_WEIGHT_SOURCE_FOLDER}/{shard}" for shard in shards]
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Could not resolve the audiogen dedup shards for %s from %s (%s); downloading all full-checkpoint shards",
|
||||
model,
|
||||
_AUDIOGEN_INDEX_FILE,
|
||||
exc,
|
||||
)
|
||||
return [f"{_WEIGHT_SOURCE_FOLDER}/model-*.safetensors"]
|
||||
|
||||
|
||||
def ensure_audex_snapshot(model: str, profile: str = "tts") -> str:
|
||||
"""Resolve ``model`` to a local repo-root directory, downloading if needed.
|
||||
|
||||
Local paths pass through untouched. For an HF repo id, download (or reuse
|
||||
from cache) exactly the subset the requesting stage's ``profile`` needs,
|
||||
so per-stage ``model_subdir`` joins land on a real snapshot instead of
|
||||
the repo-id string on a fresh cache. Falls back to a cached snapshot
|
||||
when offline.
|
||||
"""
|
||||
if os.path.isdir(model):
|
||||
return model
|
||||
|
||||
patterns = _SNAPSHOT_PROFILE_PATTERNS.get(profile)
|
||||
if patterns is None:
|
||||
raise ValueError(
|
||||
f"Unknown Audex snapshot profile {profile!r}; expected one of {sorted(_SNAPSHOT_PROFILE_PATTERNS)}"
|
||||
)
|
||||
|
||||
patterns = list(patterns)
|
||||
if profile in ("tts", "tta"):
|
||||
patterns += _dedup_shard_patterns(model)
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
try:
|
||||
return snapshot_download(model, allow_patterns=patterns)
|
||||
except Exception as download_exc:
|
||||
try:
|
||||
return snapshot_download(model, allow_patterns=patterns, local_files_only=True)
|
||||
except Exception:
|
||||
raise RuntimeError(
|
||||
f"Could not resolve the Audex repo {model!r} (profile {profile!r}): the download "
|
||||
"failed and no cached snapshot with the required folders was found."
|
||||
) from download_exc
|
||||
|
||||
|
||||
def ensure_xcodec1_snapshot(model: str | None) -> str:
|
||||
"""Resolve the external XCodec1 checkpoint (TTA waveform decoder).
|
||||
|
||||
Accepts a local directory or an HF repo id; ``None``/empty falls back to
|
||||
the official ``hf-audio/xcodec-hubert-general-balanced``.
|
||||
"""
|
||||
model = model or XCODEC1_DEFAULT_REPO
|
||||
if os.path.isdir(model):
|
||||
return model
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
try:
|
||||
return snapshot_download(model)
|
||||
except Exception as download_exc:
|
||||
try:
|
||||
return snapshot_download(model, local_files_only=True)
|
||||
except Exception:
|
||||
raise RuntimeError(
|
||||
f"Could not resolve the XCodec1 checkpoint {model!r}: the download failed and "
|
||||
"no cached snapshot was found. Download it manually or point the TTA deploy "
|
||||
"yaml's stage-1 model at a local copy."
|
||||
) from download_exc
|
||||
|
||||
|
||||
def ensure_audiogen_weights(model_dir: str) -> None:
|
||||
"""Link index-referenced shards missing from ``model_dir`` from the full checkpoint.
|
||||
|
||||
No-op when the shards are already present (or the layout is unexpected);
|
||||
raises only if a referenced shard exists nowhere.
|
||||
"""
|
||||
index_path = os.path.join(model_dir, "model.safetensors.index.json")
|
||||
if not os.path.isfile(index_path):
|
||||
return
|
||||
try:
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f).get("weight_map", {})
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("Could not read %s: %s", index_path, exc)
|
||||
return
|
||||
|
||||
source_dir = os.path.join(os.path.dirname(os.path.abspath(model_dir)), _WEIGHT_SOURCE_FOLDER)
|
||||
for shard in sorted(set(weight_map.values())):
|
||||
dst = os.path.join(model_dir, shard)
|
||||
if os.path.exists(dst):
|
||||
continue
|
||||
src = os.path.join(source_dir, shard)
|
||||
if not os.path.exists(src):
|
||||
raise FileNotFoundError(
|
||||
f"Audex checkpoint shard {shard!r} is missing from {model_dir} and "
|
||||
f"{source_dir}. Download the repo's {_WEIGHT_SOURCE_FOLDER}/ folder "
|
||||
"(it holds the deduplicated weight shards)."
|
||||
)
|
||||
try:
|
||||
os.symlink(os.path.relpath(src, model_dir), dst)
|
||||
logger.info("Linked Audex weight shard %s -> %s", dst, src)
|
||||
except OSError:
|
||||
shutil.copy2(src, dst)
|
||||
logger.info("Copied Audex weight shard %s -> %s", src, dst)
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex (Nemotron-Labs-Audex-2B) TTS pipeline topology.
|
||||
|
||||
Stage 0: Thinker — ChatML text prompt → <speechcodec_N> tokens (LLM AR).
|
||||
Stage 1: Code2Wav — streaming causal speech decoder → 16 kHz waveform.
|
||||
|
||||
Users pass the HF repo ROOT (its manifest config.json carries
|
||||
``model_type: nemotron_labs_audex``); per-stage checkpoints resolve through
|
||||
``model_subdir``/``tokenizer_subdir``. The decoder subfolder has no tokenizer
|
||||
files, so stage 1's tokenizer also points at the thinker checkpoint.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.audex"
|
||||
|
||||
# <speechgen_end> token id in checkpoint_folder_audiogen's tokenizer; pinned
|
||||
# by a unit test against the real tokenizer fixture.
|
||||
AUDEX_SPEECHGEN_END_TOKEN_ID = 131076
|
||||
|
||||
# <audiogen_end> stops TTA generation (pinned in tests via models/audex/tta.py).
|
||||
AUDEX_AUDIOGEN_END_TOKEN_ID = 131074
|
||||
|
||||
# No pipeline-level model_arch: stage-0 classes resolve from each
|
||||
# checkpoint's own ``architectures`` (dense on the 2B, NemotronH on the
|
||||
# 30B-A3B); forcing one name via hf_overrides would misbind the other
|
||||
# size. Decoder stages keep their explicit per-stage model_arch.
|
||||
AUDEX_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="audex_tts",
|
||||
default_deploy_config_name="audex_tts.yaml",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audex_thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
model_subdir="checkpoint_folder_audiogen",
|
||||
tokenizer_subdir="checkpoint_folder_audiogen",
|
||||
# Emits the unconditional CFG companion for cfg_scale > 1.0
|
||||
# requests; a no-op for unguided requests.
|
||||
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.thinker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [AUDEX_SPEECHGEN_END_TOKEN_ID],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audex_code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="AudexCode2Wav",
|
||||
model_subdir="audex_causal_speech_decoder",
|
||||
tokenizer_subdir="checkpoint_folder_audiogen",
|
||||
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Text-to-audio (caption → general audio): same audiogen thinker checkpoint,
|
||||
# but the output rides the interleaved 4-codebook <audiocodec_N> RVQ block
|
||||
# and decodes through the EXTERNAL XCodec1 checkpoint. XCodec1 is a CNN codec
|
||||
# decoded over the full sequence, so this pipeline is sync full-payload only
|
||||
# (deploy yaml sets async_chunk: false). CFG is effectively mandatory for TTA
|
||||
# quality (official default scale 3.0).
|
||||
# Cascaded speech-to-speech: the audio-capable full checkpoint serves all
|
||||
# three official passes (ASR → chat → TTS) in one deployment; only the TTS
|
||||
# pass (output modality "audio", final_stage_id 1) traverses the streaming
|
||||
# speech decoder — text-modality passes finish at stage 0 and never reach
|
||||
# the codec path. The cascade itself is orchestrated by the client/example.
|
||||
AUDEX_S2S_PIPELINE = PipelineConfig(
|
||||
model_type="audex_s2s",
|
||||
default_deploy_config_name="audex_s2s.yaml",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audex_omni",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="latent",
|
||||
model_subdir="checkpoint_folder_full",
|
||||
tokenizer_subdir="checkpoint_folder_full",
|
||||
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.thinker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2code2wav_full_payload",
|
||||
# <speechgen_end> additionally stops the TTS pass; text passes
|
||||
# stop at their natural eos first. detokenize stays on for the
|
||||
# text-final passes.
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [AUDEX_SPEECHGEN_END_TOKEN_ID],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audex_code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="AudexCode2Wav",
|
||||
model_subdir="audex_causal_speech_decoder",
|
||||
# The decoder folder has no tokenizer files. Borrow the audiogen
|
||||
# tokenizer (not checkpoint_folder_full): this stage resolves its
|
||||
# snapshot with the "tts" download profile, which guarantees
|
||||
# checkpoint_folder_audiogen but not the full folder's tokenizer,
|
||||
# so per-stage / clean-cache launches would otherwise fail.
|
||||
tokenizer_subdir="checkpoint_folder_audiogen",
|
||||
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Audio understanding (ASR / audio QA): single-stage deployment of the
|
||||
# audio-capable full checkpoint — speech (+ text instruction) in, text out.
|
||||
# Modeled on the ming_flash_omni / qwen2_5_omni thinker-only pipelines.
|
||||
AUDEX_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
model_type="audex_thinker_only",
|
||||
default_deploy_config_name="audex_thinker_only.yaml",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audex_omni",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
engine_output_type="text",
|
||||
model_subdir="checkpoint_folder_full",
|
||||
tokenizer_subdir="checkpoint_folder_full",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
AUDEX_TTA_PIPELINE = PipelineConfig(
|
||||
model_type="audex_tta",
|
||||
default_deploy_config_name="audex_tta.yaml",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audex_tta_thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
model_subdir="checkpoint_folder_audiogen",
|
||||
tokenizer_subdir="checkpoint_folder_audiogen",
|
||||
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2xcodec_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [AUDEX_AUDIOGEN_END_TOKEN_ID],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audex_xcodec",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="AudexXCodec1",
|
||||
# Length placeholder for the stage-1 request; the bulk codec
|
||||
# payload ships via thinker2xcodec_full_payload.
|
||||
sync_process_input_func=f"{_PROC}.thinker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex TTS/TTA prompt construction.
|
||||
|
||||
The Audex thinker consumes the exact ChatML prompt used by the official
|
||||
inference script (``inference_scripts_vllm/audiogen_scripts/run_audio_gen_vllm.py``
|
||||
in the nvidia/Nemotron-Labs-Audex-2B repo). The checkpoint's bundled chat
|
||||
template opens a thinking block (``<think>\\n``) instead of the closed
|
||||
``<think></think>`` + ``<speechgen_start>`` priming that TTS generation
|
||||
requires, so the prompt is built from a literal template here; a unit test
|
||||
pins it byte-for-byte against the official format.
|
||||
|
||||
``build_null_prompt`` is the unconditional-prompt counterpart used by
|
||||
classifier-free guidance: the transcription is replaced with repeated
|
||||
``<unk>`` tokens, iteratively adjusted so the tokenized null prompt is
|
||||
exactly as long as the conditional prompt (the CFG pair must decode the
|
||||
same positions).
|
||||
"""
|
||||
|
||||
AUDEX_SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
|
||||
|
||||
# ``{text}`` carries its own leading whitespace: the conditional prompt uses
|
||||
# " {transcription}" (matching the official script byte-for-byte), while the
|
||||
# CFG null prompt inserts the bare ``<unk>`` run. ``<unk>`` is a special
|
||||
# token, so a leading space would tokenize as a standalone space token and
|
||||
# make some conditional lengths unreachable by whole-``<unk>`` steps.
|
||||
_TTS_PROMPT_TEMPLATE = (
|
||||
"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|text to speech|> Generate speech for this transcription.{text}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n<think></think><speechgen_start>"
|
||||
)
|
||||
|
||||
_TTA_PROMPT_TEMPLATE = (
|
||||
"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|text to audio|> Generate audio for this caption.{text}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n<think></think><audiogen_start>"
|
||||
)
|
||||
|
||||
|
||||
def build_cond_prompt(text: str) -> str:
|
||||
"""Build the conditional TTS prompt for one transcription."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("Audex TTS requires non-empty input text")
|
||||
return _TTS_PROMPT_TEMPLATE.format(system_prompt=AUDEX_SYSTEM_PROMPT, text=f" {text}")
|
||||
|
||||
|
||||
def build_tta_cond_prompt(caption: str) -> str:
|
||||
"""Build the conditional TTA prompt for one audio caption."""
|
||||
caption = caption.strip()
|
||||
if not caption:
|
||||
raise ValueError("Audex TTA requires a non-empty caption")
|
||||
return _TTA_PROMPT_TEMPLATE.format(system_prompt=AUDEX_SYSTEM_PROMPT, text=f" {caption}")
|
||||
|
||||
|
||||
def _length_matched_null(cond_prompt: str, tokenizer, template: str, max_iters: int) -> str:
|
||||
target_len = len(tokenizer.encode(cond_prompt))
|
||||
n_unk = 1
|
||||
for _ in range(max_iters):
|
||||
prompt = template.format(system_prompt=AUDEX_SYSTEM_PROMPT, text="<unk>" * n_unk)
|
||||
diff = target_len - len(tokenizer.encode(prompt))
|
||||
if diff == 0:
|
||||
return prompt
|
||||
n_unk = max(1, n_unk + diff)
|
||||
if diff < 0 and n_unk == 1:
|
||||
break
|
||||
raise ValueError(f"Could not length-match the CFG null prompt to {target_len} tokens")
|
||||
|
||||
|
||||
def build_null_prompt(cond_prompt: str, tokenizer, max_iters: int = 64) -> str:
|
||||
"""Unconditional TTS prompt for CFG (length-matched ``<unk>`` padding).
|
||||
|
||||
Replaces the transcription with ``<unk>`` repeated, adjusting the count
|
||||
until ``tokenizer.encode`` yields exactly the conditional prompt's
|
||||
length. Raises when no count matches so callers can fail the request
|
||||
instead of submitting a misaligned pair.
|
||||
"""
|
||||
return _length_matched_null(cond_prompt, tokenizer, _TTS_PROMPT_TEMPLATE, max_iters)
|
||||
|
||||
|
||||
def build_tta_null_prompt(cond_prompt: str, tokenizer, max_iters: int = 64) -> str:
|
||||
"""Unconditional TTA prompt for CFG (length-matched ``<unk>`` padding)."""
|
||||
return _length_matched_null(cond_prompt, tokenizer, _TTA_PROMPT_TEMPLATE, max_iters)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
"""Vendored Audex causal speech decoder (Apache-2.0).
|
||||
|
||||
Verbatim copy of the ``audex_causal_speech_decoder`` remote-code package from
|
||||
nvidia/Nemotron-Labs-Audex-2B. Vendored because transformers' dynamic module
|
||||
loader resolves HuggingFace-cache blob symlinks to their real paths, which
|
||||
breaks the package's relative imports when loading with trust_remote_code
|
||||
directly from a hub snapshot.
|
||||
"""
|
||||
|
||||
from .configuration_audex_causal_speech_decoder import AudexCausalSpeechDecoderConfig
|
||||
from .modeling_audex_causal_speech_decoder import (
|
||||
AudexCausalSpeechDecoderModel,
|
||||
AudexCausalSpeechDecoderSession,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AudexCausalSpeechDecoderConfig",
|
||||
"AudexCausalSpeechDecoderModel",
|
||||
"AudexCausalSpeechDecoderSession",
|
||||
]
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
|
||||
class AudexCausalSpeechDecoderConfig(PretrainedConfig):
|
||||
model_type = "audex_causal_speech_decoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_dim: int = 2048,
|
||||
depth: int = 12,
|
||||
heads: int = 32,
|
||||
pos_meb_dim: int = 64,
|
||||
hop_length: int = 320,
|
||||
vq_dim: int = 2048,
|
||||
lookahead_steps: int = 4,
|
||||
sample_rate: int = 16000,
|
||||
codebook_levels: list[int] | None = None,
|
||||
codebook_size: int = 65536,
|
||||
token_embed_dim: int = 8,
|
||||
embed_tokens_from_codes: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.hidden_dim = hidden_dim
|
||||
self.depth = depth
|
||||
self.heads = heads
|
||||
self.pos_meb_dim = pos_meb_dim
|
||||
self.hop_length = hop_length
|
||||
self.vq_dim = vq_dim
|
||||
self.lookahead_steps = lookahead_steps
|
||||
self.sample_rate = sample_rate
|
||||
self.codebook_levels = codebook_levels or [4, 4, 4, 4, 4, 4, 4, 4]
|
||||
self.codebook_size = codebook_size
|
||||
self.token_embed_dim = token_embed_dim
|
||||
self.embed_tokens_from_codes = embed_tokens_from_codes
|
||||
+478
@@ -0,0 +1,478 @@
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from .configuration_audex_causal_speech_decoder import AudexCausalSpeechDecoderConfig
|
||||
from .streaming_utils import load_audex_causal_speech_decoder as _load_decoder_for_remote_code
|
||||
|
||||
REMOTE_CODE_IMPORTS = (_load_decoder_for_remote_code,)
|
||||
|
||||
|
||||
class RotaryPositionalEmbeddings(nn.Module):
|
||||
def __init__(self, dim: int, max_seq_len: int = 4096, base: int = 10_000) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.base = base
|
||||
self.max_seq_len = max_seq_len
|
||||
self.rope_init()
|
||||
self._rope_ready = False
|
||||
|
||||
def rope_init(self, device: torch.device | None = None) -> None:
|
||||
theta = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, device=device)[: (self.dim // 2)].float() / self.dim))
|
||||
self.register_buffer("theta", theta, persistent=False)
|
||||
self.build_rope_cache(self.max_seq_len)
|
||||
|
||||
def build_rope_cache(self, max_seq_len: int = 4096) -> None:
|
||||
self.max_seq_len = max_seq_len
|
||||
seq_idx = torch.arange(max_seq_len, dtype=self.theta.dtype, device=self.theta.device)
|
||||
idx_theta = torch.einsum("i, j -> ij", seq_idx, self.theta).float()
|
||||
cache = torch.stack([torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1)
|
||||
self.register_buffer("cache", cache, persistent=False)
|
||||
|
||||
def forward(self, x: torch.Tensor, *, input_pos: torch.Tensor | None = None) -> torch.Tensor:
|
||||
seq_len = x.size(1)
|
||||
needed_seq_len = seq_len if input_pos is None else int(input_pos.max().item()) + 1
|
||||
if (
|
||||
not getattr(self, "_rope_ready", False)
|
||||
or self.theta.device != x.device
|
||||
or needed_seq_len > self.cache.size(0)
|
||||
):
|
||||
self.rope_init(device=x.device)
|
||||
if needed_seq_len > self.cache.size(0):
|
||||
self.build_rope_cache(max(needed_seq_len, self.cache.size(0) * 2))
|
||||
self._rope_ready = True
|
||||
|
||||
rope_cache = self.cache[:seq_len] if input_pos is None else self.cache[input_pos]
|
||||
xshaped = x.float().reshape(*x.shape[:-1], -1, 2)
|
||||
rope_cache = rope_cache.view(-1, xshaped.size(1), 1, xshaped.size(3), 2)
|
||||
x_out = torch.stack(
|
||||
[
|
||||
xshaped[..., 0] * rope_cache[..., 0] - xshaped[..., 1] * rope_cache[..., 1],
|
||||
xshaped[..., 1] * rope_cache[..., 0] + xshaped[..., 0] * rope_cache[..., 1],
|
||||
],
|
||||
-1,
|
||||
)
|
||||
return x_out.flatten(3).type_as(x)
|
||||
|
||||
|
||||
class CausalCodecDecoderCache:
|
||||
def __init__(self) -> None:
|
||||
self.key_values: dict[int, tuple[Tensor, Tensor]] = {}
|
||||
self.position = 0
|
||||
|
||||
def input_positions(self, length: int, device: torch.device) -> Tensor:
|
||||
return torch.arange(self.position, self.position + length, device=device).unsqueeze(0)
|
||||
|
||||
def update(self, layer_idx: int, key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]:
|
||||
if layer_idx in self.key_values:
|
||||
prev_key, prev_value = self.key_values[layer_idx]
|
||||
key = torch.cat([prev_key, key], dim=2)
|
||||
value = torch.cat([prev_value, value], dim=2)
|
||||
self.key_values[layer_idx] = (key, value)
|
||||
return key, value
|
||||
|
||||
def advance(self, length: int) -> None:
|
||||
self.position += length
|
||||
|
||||
def reset(self) -> None:
|
||||
self.key_values.clear()
|
||||
self.position = 0
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
norm_x = torch.mean(x**2, dim=-1, keepdim=True)
|
||||
return x * torch.rsqrt(norm_x + self.eps) * self.weight
|
||||
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, dim: int) -> None:
|
||||
super().__init__()
|
||||
self.fc1 = nn.Linear(dim, 4 * dim, bias=False)
|
||||
self.silu = nn.SiLU()
|
||||
self.fc2 = nn.Linear(4 * dim, dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.fc2(self.silu(self.fc1(x)))
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(self, dim: int, n_heads: int, rotary_embed: RotaryPositionalEmbeddings, layer_idx: int):
|
||||
super().__init__()
|
||||
if dim % n_heads != 0:
|
||||
raise ValueError(f"dim must be divisible by n_heads, got dim={dim}, n_heads={n_heads}")
|
||||
self.n_heads = n_heads
|
||||
self.layer_idx = layer_idx
|
||||
self.rotary_embed = rotary_embed
|
||||
self.c_attn = nn.Linear(dim, 3 * dim, bias=False)
|
||||
self.c_proj = nn.Linear(dim, dim, bias=False)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cache: CausalCodecDecoderCache | None = None,
|
||||
input_pos: Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
batch_size, seq_len, _ = x.shape
|
||||
qkv = self.c_attn(x)
|
||||
head_dim = qkv.size(-1) // (3 * self.n_heads)
|
||||
qkv = qkv.view(batch_size, seq_len, 3, self.n_heads, head_dim).permute(2, 0, 3, 1, 4)
|
||||
q, k, v = qkv.unbind(0)
|
||||
|
||||
q = self.rotary_embed(q.transpose(1, 2), input_pos=input_pos).transpose(1, 2)
|
||||
k = self.rotary_embed(k.transpose(1, 2), input_pos=input_pos).transpose(1, 2)
|
||||
if cache is None:
|
||||
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
|
||||
else:
|
||||
if input_pos is None:
|
||||
raise ValueError("input_pos is required when cache is set")
|
||||
k, v = cache.update(self.layer_idx, k, v)
|
||||
key_pos = torch.arange(k.size(2), device=x.device).view(1, 1, 1, -1)
|
||||
attn_mask = key_pos <= input_pos.view(input_pos.size(0), 1, -1, 1)
|
||||
y = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask)
|
||||
return self.c_proj(y.transpose(1, 2).contiguous().view(batch_size, seq_len, -1))
|
||||
|
||||
|
||||
class TransformerBlock(nn.Module):
|
||||
def __init__(self, dim: int, n_heads: int, rotary_embed: RotaryPositionalEmbeddings, layer_idx: int):
|
||||
super().__init__()
|
||||
self.att_norm = RMSNorm(dim)
|
||||
self.ffn_norm = RMSNorm(dim)
|
||||
self.att = Attention(dim=dim, n_heads=n_heads, rotary_embed=rotary_embed, layer_idx=layer_idx)
|
||||
self.mlp = MLP(dim=dim)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cache: CausalCodecDecoderCache | None = None,
|
||||
input_pos: Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = x + self.att(self.att_norm(x), cache=cache, input_pos=input_pos)
|
||||
return x + self.mlp(self.ffn_norm(x))
|
||||
|
||||
|
||||
class PatchHead(nn.Module):
|
||||
def __init__(self, dim: int, hop_length: int = 320):
|
||||
super().__init__()
|
||||
self.proj = nn.Linear(dim, hop_length, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = torch.tanh(self.proj(x))
|
||||
return x.reshape(x.size(0), 1, -1)
|
||||
|
||||
|
||||
class CausalVocosBackbone(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_dim: int = 2048,
|
||||
depth: int = 12,
|
||||
heads: int = 32,
|
||||
pos_meb_dim: int = 64,
|
||||
):
|
||||
super().__init__()
|
||||
rotary_embed = RotaryPositionalEmbeddings(dim=pos_meb_dim)
|
||||
self.transformers = nn.ModuleList(
|
||||
[
|
||||
TransformerBlock(dim=hidden_dim, n_heads=heads, rotary_embed=rotary_embed, layer_idx=idx)
|
||||
for idx in range(depth)
|
||||
]
|
||||
)
|
||||
self.final_layer_norm = RMSNorm(hidden_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, cache: CausalCodecDecoderCache | None = None) -> torch.Tensor:
|
||||
input_pos = None
|
||||
if cache is not None:
|
||||
input_pos = cache.input_positions(x.size(1), x.device).expand(x.size(0), -1)
|
||||
for block in self.transformers:
|
||||
x = block(x, cache=cache, input_pos=input_pos)
|
||||
if cache is not None:
|
||||
cache.advance(x.size(1))
|
||||
return self.final_layer_norm(x)
|
||||
|
||||
|
||||
class CausalCodecDecoderVocos(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_dim: int = 2048,
|
||||
depth: int = 12,
|
||||
heads: int = 32,
|
||||
pos_meb_dim: int = 64,
|
||||
hop_length: int = 320,
|
||||
vq_dim: int = 2048,
|
||||
lookahead_steps: int = 0,
|
||||
):
|
||||
super().__init__()
|
||||
if lookahead_steps < 0:
|
||||
raise ValueError(f"lookahead_steps must be >= 0, got {lookahead_steps}")
|
||||
|
||||
self.wav_proj = nn.Linear(hop_length, hidden_dim, bias=False)
|
||||
self.fc_post_a = nn.Linear(vq_dim, hidden_dim, bias=False)
|
||||
self.lookahead_steps = lookahead_steps
|
||||
if lookahead_steps > 0:
|
||||
self.lookahead_conv = nn.Conv1d(
|
||||
hidden_dim,
|
||||
hidden_dim,
|
||||
kernel_size=lookahead_steps + 1,
|
||||
padding=0,
|
||||
groups=hidden_dim,
|
||||
bias=False,
|
||||
)
|
||||
self.lookahead_act = nn.SiLU()
|
||||
self.lookahead_proj = nn.Conv1d(hidden_dim, hidden_dim, kernel_size=1, bias=False)
|
||||
nn.init.zeros_(self.lookahead_proj.weight)
|
||||
else:
|
||||
self.lookahead_conv = None
|
||||
self.lookahead_act = None
|
||||
self.lookahead_proj = None
|
||||
self.backbone = CausalVocosBackbone(hidden_dim, depth, heads, pos_meb_dim)
|
||||
self.head = PatchHead(hidden_dim, hop_length)
|
||||
|
||||
def _project_tokens(self, vq_emb: torch.Tensor) -> torch.Tensor:
|
||||
return self.fc_post_a(vq_emb)
|
||||
|
||||
def _apply_lookahead(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.lookahead_conv is None:
|
||||
return x
|
||||
if self.lookahead_act is None or self.lookahead_proj is None:
|
||||
raise RuntimeError("lookahead modules are not initialized")
|
||||
h = F.pad(x.transpose(1, 2), (0, self.lookahead_steps))
|
||||
h = self.lookahead_proj(self.lookahead_act(self.lookahead_conv(h)))
|
||||
return x + h.transpose(1, 2)
|
||||
|
||||
def _apply_lookahead_window(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.lookahead_conv is None:
|
||||
return x
|
||||
if self.lookahead_act is None or self.lookahead_proj is None:
|
||||
raise RuntimeError("lookahead modules are not initialized")
|
||||
if x.size(1) <= self.lookahead_steps:
|
||||
raise ValueError(f"lookahead window needs more than {self.lookahead_steps} frames, got {x.size(1)}")
|
||||
h = self.lookahead_proj(self.lookahead_act(self.lookahead_conv(x.transpose(1, 2))))
|
||||
return x[:, : h.size(2)] + h.transpose(1, 2)
|
||||
|
||||
def decode_cached(
|
||||
self,
|
||||
vq_emb: torch.Tensor,
|
||||
cache: CausalCodecDecoderCache,
|
||||
lookahead_vq_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self._project_tokens(vq_emb)
|
||||
if self.lookahead_steps > 0:
|
||||
if lookahead_vq_emb is None:
|
||||
lookahead_vq_emb = vq_emb.new_zeros(vq_emb.size(0), self.lookahead_steps, vq_emb.size(-1))
|
||||
if lookahead_vq_emb.size(1) != self.lookahead_steps:
|
||||
raise ValueError(
|
||||
f"lookahead_vq_emb must have {self.lookahead_steps} frames, got {lookahead_vq_emb.size(1)}"
|
||||
)
|
||||
lookahead_x = self._project_tokens(lookahead_vq_emb)
|
||||
x = self._apply_lookahead_window(torch.cat([x, lookahead_x], dim=1))
|
||||
x = self.backbone(x, cache=cache)
|
||||
return self.head(x)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
vq_emb: torch.Tensor,
|
||||
patched_wav: torch.Tensor | None = None,
|
||||
alpha: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
x = self._project_tokens(vq_emb)
|
||||
x = self._apply_lookahead(x)
|
||||
if patched_wav is not None:
|
||||
h = self.wav_proj(patched_wav)
|
||||
mask = torch.bernoulli(
|
||||
torch.full(
|
||||
(x.size(0), x.size(1), 1),
|
||||
min(max(alpha, 0.0), 1.0),
|
||||
device=x.device,
|
||||
dtype=x.dtype,
|
||||
)
|
||||
)
|
||||
x = x + h * mask
|
||||
return self.head(self.backbone(x))
|
||||
|
||||
|
||||
class AudexSpeechTokenEmbedder(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
output_dim: int,
|
||||
token_embed_dim: int,
|
||||
codebook_levels: Sequence[int],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if len(codebook_levels) != token_embed_dim:
|
||||
raise ValueError(
|
||||
f"token_embed_dim={token_embed_dim} must match codebook_levels length={len(codebook_levels)}"
|
||||
)
|
||||
self.codebook_levels = tuple(int(level) for level in codebook_levels)
|
||||
self.project_out = nn.Linear(token_embed_dim, output_dim)
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
if indices.size(-1) != 1:
|
||||
raise ValueError(f"indices last dimension must be 1, got {indices.size(-1)}")
|
||||
levels = torch.tensor(self.codebook_levels, dtype=torch.long, device=indices.device)
|
||||
basis = torch.cumprod(torch.cat([levels.new_ones(1), levels[:-1]]), dim=0)
|
||||
level_indices = (indices.long() // basis) % levels
|
||||
dtype = self.project_out.weight.dtype
|
||||
codes = level_indices.to(dtype=dtype)
|
||||
levels = levels.to(dtype=dtype)
|
||||
codes = codes * (2.0 / (levels - 1.0)) - 1.0
|
||||
return self.project_out(codes)
|
||||
|
||||
def get_output_from_indices(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
return self(indices)
|
||||
|
||||
|
||||
class AudexCausalSpeechDecoderModel(PreTrainedModel):
|
||||
config_class = AudexCausalSpeechDecoderConfig
|
||||
base_model_prefix = "module"
|
||||
all_tied_weights_keys: dict[str, Any] = {}
|
||||
Cache = CausalCodecDecoderCache
|
||||
|
||||
def __init__(self, config: AudexCausalSpeechDecoderConfig):
|
||||
super().__init__(config)
|
||||
self.audex_speech_token_embedder = AudexSpeechTokenEmbedder(
|
||||
output_dim=config.vq_dim,
|
||||
token_embed_dim=config.token_embed_dim,
|
||||
codebook_levels=config.codebook_levels,
|
||||
)
|
||||
self.module = CausalCodecDecoderVocos(
|
||||
hidden_dim=config.hidden_dim,
|
||||
depth=config.depth,
|
||||
heads=config.heads,
|
||||
pos_meb_dim=config.pos_meb_dim,
|
||||
hop_length=config.hop_length,
|
||||
vq_dim=config.vq_dim,
|
||||
lookahead_steps=config.lookahead_steps,
|
||||
)
|
||||
|
||||
@property
|
||||
def lookahead_steps(self) -> int:
|
||||
return self.module.lookahead_steps
|
||||
|
||||
def create_cache(self) -> CausalCodecDecoderCache:
|
||||
return CausalCodecDecoderCache()
|
||||
|
||||
def decode_cached(
|
||||
self,
|
||||
vq_emb: torch.Tensor,
|
||||
cache: CausalCodecDecoderCache,
|
||||
lookahead_vq_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.module.decode_cached(vq_emb, cache, lookahead_vq_emb=lookahead_vq_emb)
|
||||
|
||||
def create_session(
|
||||
self,
|
||||
*,
|
||||
chunk_frames: int = 1,
|
||||
sample_rate: int | None = None,
|
||||
return_numpy: bool = True,
|
||||
) -> AudexCausalSpeechDecoderSession:
|
||||
return AudexCausalSpeechDecoderSession(
|
||||
decoder=self,
|
||||
chunk_frames=chunk_frames,
|
||||
sample_rate=sample_rate or self.config.sample_rate,
|
||||
return_numpy=return_numpy,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
vq_emb: torch.Tensor,
|
||||
patched_wav: torch.Tensor | None = None,
|
||||
alpha: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
return self.module(vq_emb, patched_wav=patched_wav, alpha=alpha)
|
||||
|
||||
|
||||
class AudexCausalSpeechDecoderSession:
|
||||
def __init__(
|
||||
self,
|
||||
decoder: AudexCausalSpeechDecoderModel,
|
||||
*,
|
||||
chunk_frames: int,
|
||||
sample_rate: int,
|
||||
return_numpy: bool,
|
||||
):
|
||||
if chunk_frames <= 0:
|
||||
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
|
||||
self.decoder = decoder
|
||||
self.chunk_frames = chunk_frames
|
||||
self.sample_rate = sample_rate
|
||||
self.return_numpy = return_numpy
|
||||
self.cache = decoder.create_cache()
|
||||
self.buffer: list[list[int]] = []
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return next(self.decoder.parameters()).device
|
||||
|
||||
def reset(self) -> None:
|
||||
self.cache = self.decoder.create_cache()
|
||||
self.buffer.clear()
|
||||
|
||||
def push(self, token_frames: Sequence[Sequence[int]]) -> Iterator[tuple[int, Any]]:
|
||||
self.buffer.extend(list(frame) for frame in token_frames)
|
||||
yield from self._drain(flush=False)
|
||||
|
||||
def flush(self) -> Iterator[tuple[int, Any]]:
|
||||
yield from self._drain(flush=True)
|
||||
|
||||
def _drain(self, *, flush: bool) -> Iterator[tuple[int, Any]]:
|
||||
ready_frames = len(self.buffer) - self.decoder.lookahead_steps
|
||||
while self.buffer and (flush or ready_frames >= self.chunk_frames):
|
||||
emit_frames = min(self.chunk_frames, len(self.buffer)) if flush else self.chunk_frames
|
||||
wav = self._decode_buffered_frames(emit_frames, flush=flush)
|
||||
del self.buffer[:emit_frames]
|
||||
ready_frames = len(self.buffer) - self.decoder.lookahead_steps
|
||||
yield self.sample_rate, self._format_chunk(wav)
|
||||
|
||||
def _embed_speech_token_frames(self, token_frames: Sequence[Sequence[int]]) -> torch.Tensor:
|
||||
indices = torch.tensor(token_frames, dtype=torch.long, device=self.device).unsqueeze(0)
|
||||
return self.decoder.audex_speech_token_embedder.get_output_from_indices(indices)
|
||||
|
||||
def _decode_buffered_frames(self, emit_frames: int, *, flush: bool) -> torch.Tensor:
|
||||
with torch.inference_mode():
|
||||
vq_emb = self._embed_speech_token_frames(self.buffer[:emit_frames])
|
||||
lookahead_vq_emb = None
|
||||
if self.decoder.lookahead_steps > 0:
|
||||
future_frames = self.buffer[emit_frames : emit_frames + self.decoder.lookahead_steps]
|
||||
future_parts = []
|
||||
if future_frames:
|
||||
future_parts.append(self._embed_speech_token_frames(future_frames))
|
||||
missing_frames = self.decoder.lookahead_steps - len(future_frames) if flush else 0
|
||||
if missing_frames > 0:
|
||||
future_parts.append(vq_emb.new_zeros(vq_emb.size(0), missing_frames, vq_emb.size(-1)))
|
||||
lookahead_vq_emb = torch.cat(future_parts, dim=1) if future_parts else None
|
||||
return self.decoder.decode_cached(vq_emb, self.cache, lookahead_vq_emb=lookahead_vq_emb)
|
||||
|
||||
def _format_chunk(self, wav: torch.Tensor) -> Any:
|
||||
chunk = wav.squeeze().float().detach().cpu()
|
||||
if not self.return_numpy:
|
||||
return chunk
|
||||
|
||||
import numpy as np
|
||||
|
||||
return chunk.numpy().astype(np.float32, copy=False)
|
||||
@@ -0,0 +1,140 @@
|
||||
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .modeling_audex_causal_speech_decoder import AudexCausalSpeechDecoderModel
|
||||
|
||||
|
||||
def freeze_model(model: torch.nn.Module) -> torch.nn.Module:
|
||||
model.eval()
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
return model
|
||||
|
||||
|
||||
def load_audex_causal_speech_decoder(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device | str = "cuda",
|
||||
dtype: torch.dtype | str | None = None,
|
||||
) -> AudexCausalSpeechDecoderModel:
|
||||
kwargs = {"trust_remote_code": True}
|
||||
if dtype is not None:
|
||||
kwargs["torch_dtype"] = dtype
|
||||
model = AutoModel.from_pretrained(model_path, **kwargs)
|
||||
return freeze_model(model.to(device))
|
||||
|
||||
|
||||
def _decoder_device(decoder: AudexCausalSpeechDecoderModel) -> torch.device:
|
||||
return next(decoder.parameters()).device
|
||||
|
||||
|
||||
def _embed_speech_token_frames(
|
||||
decoder: AudexCausalSpeechDecoderModel,
|
||||
token_frames: Sequence[Sequence[int]],
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
indices = torch.tensor(token_frames, dtype=torch.long, device=device).unsqueeze(0)
|
||||
return decoder.audex_speech_token_embedder.get_output_from_indices(indices)
|
||||
|
||||
|
||||
def decode_speech_token_frames(
|
||||
token_frames: Iterable[Sequence[int]],
|
||||
decoder: AudexCausalSpeechDecoderModel,
|
||||
*,
|
||||
device: torch.device | str | None = None,
|
||||
chunk_frames: int = 1,
|
||||
) -> Iterator[torch.Tensor]:
|
||||
if chunk_frames <= 0:
|
||||
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
|
||||
|
||||
device = torch.device(device) if device is not None else _decoder_device(decoder)
|
||||
cache = decoder.create_cache()
|
||||
buffer: list[list[int]] = []
|
||||
lookahead_steps = decoder.lookahead_steps
|
||||
|
||||
for token_frame in token_frames:
|
||||
buffer.append(list(token_frame))
|
||||
while len(buffer) > lookahead_steps:
|
||||
emit_frames = min(chunk_frames, len(buffer) - lookahead_steps)
|
||||
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=False)
|
||||
del buffer[:emit_frames]
|
||||
yield wav
|
||||
|
||||
while buffer:
|
||||
emit_frames = min(chunk_frames, len(buffer))
|
||||
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=True)
|
||||
del buffer[:emit_frames]
|
||||
yield wav
|
||||
|
||||
|
||||
async def decode_speech_token_stream(
|
||||
token_stream: AsyncIterable[Sequence[int]],
|
||||
decoder: AudexCausalSpeechDecoderModel,
|
||||
*,
|
||||
device: torch.device | str | None = None,
|
||||
chunk_frames: int = 1,
|
||||
) -> AsyncIterator[torch.Tensor]:
|
||||
if chunk_frames <= 0:
|
||||
raise ValueError(f"chunk_frames must be positive, got {chunk_frames}")
|
||||
|
||||
device = torch.device(device) if device is not None else _decoder_device(decoder)
|
||||
cache = decoder.create_cache()
|
||||
buffer: list[list[int]] = []
|
||||
lookahead_steps = decoder.lookahead_steps
|
||||
|
||||
async for token_frame in token_stream:
|
||||
buffer.append(list(token_frame))
|
||||
while len(buffer) > lookahead_steps:
|
||||
emit_frames = min(chunk_frames, len(buffer) - lookahead_steps)
|
||||
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=False)
|
||||
del buffer[:emit_frames]
|
||||
yield wav
|
||||
|
||||
while buffer:
|
||||
emit_frames = min(chunk_frames, len(buffer))
|
||||
wav = _decode_buffered_frames(decoder, cache, buffer, emit_frames, device, flush=True)
|
||||
del buffer[:emit_frames]
|
||||
yield wav
|
||||
|
||||
|
||||
def _decode_buffered_frames(
|
||||
decoder: AudexCausalSpeechDecoderModel,
|
||||
cache: Any,
|
||||
buffer: Sequence[Sequence[int]],
|
||||
emit_frames: int,
|
||||
device: torch.device,
|
||||
*,
|
||||
flush: bool,
|
||||
) -> torch.Tensor:
|
||||
with torch.inference_mode():
|
||||
vq_emb = _embed_speech_token_frames(decoder, buffer[:emit_frames], device)
|
||||
lookahead_vq_emb = None
|
||||
if decoder.lookahead_steps > 0:
|
||||
future_frames = buffer[emit_frames : emit_frames + decoder.lookahead_steps]
|
||||
future_parts = []
|
||||
if future_frames:
|
||||
future_parts.append(_embed_speech_token_frames(decoder, future_frames, device))
|
||||
missing_frames = decoder.lookahead_steps - len(future_frames) if flush else 0
|
||||
if missing_frames > 0:
|
||||
future_parts.append(vq_emb.new_zeros(vq_emb.size(0), missing_frames, vq_emb.size(-1)))
|
||||
lookahead_vq_emb = torch.cat(future_parts, dim=1) if future_parts else None
|
||||
return decoder.decode_cached(vq_emb, cache, lookahead_vq_emb=lookahead_vq_emb)
|
||||
@@ -0,0 +1,293 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from nvidia/Nemotron-Labs-Audex-2B (Apache-2.0):
|
||||
# inference_scripts_vllm/audiogen_scripts/tta_rvq_logits_processor.py
|
||||
# inference_scripts_vllm/audiogen_scripts/run_audio_gen_vllm_rvq_logit_mask.py
|
||||
# Divergences from the official files are commented at the divergence site.
|
||||
"""Audex text-to-audio (TTA) token space: RVQ phase masking and helpers.
|
||||
|
||||
TTA output is an interleaved 4-codebook RVQ stream over ``<audiocodec_N>``
|
||||
tokens: generated position ``p`` (counted from ``<audiogen_start>``) must
|
||||
come from codebook ``p % 4``, whose codec-id range is
|
||||
``[phase * 1024, (phase + 1) * 1024)``. The tokenizer defines 8192
|
||||
``<audiocodec_*>`` symbols; only codec ids 0..4095 (the 4 quantizers XCodec1
|
||||
decodes) are generated — the upper half stays disallowed.
|
||||
|
||||
Unguided/unmasked sampling emits phase-invalid sequences that XCodec1 cannot
|
||||
decode, so :class:`TTARVQPhaseMaskLogitsProcessor` is required for validity,
|
||||
driven per request by ``SamplingParams.extra_args["tta_rvq"]``:
|
||||
|
||||
{
|
||||
"phase_token_ids": [[...1024 ids...] x 4],
|
||||
"start_tid": <audiogen_start id>,
|
||||
"end_tid": <audiogen_end id>,
|
||||
"codec_cap": <max codec tokens or None>,
|
||||
"start_in_prompt": <bool>,
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
BatchUpdate,
|
||||
LogitsProcessor,
|
||||
MoveDirectionality,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
NEG_INF = float("-inf")
|
||||
XCODEC1_NUM_CODEBOOKS = 4
|
||||
XCODEC1_CODEBOOK_SIZE = 1024
|
||||
|
||||
# Pinned by a unit test against the real checkpoint tokenizer.
|
||||
AUDEX_AUDIOGEN_START_TOKEN_ID = 131073
|
||||
AUDEX_AUDIOGEN_END_TOKEN_ID = 131074
|
||||
AUDEX_AUDIOCODEC_TOKEN_OFFSET = 196613
|
||||
AUDEX_AUDIOCODEC_VOCAB_SIZE = 8192
|
||||
|
||||
|
||||
def build_tta_phase_token_ids(tokenizer: Any) -> tuple[list[list[int]], int, int]:
|
||||
"""Group tokenizer ids of ``<audiocodec_N>`` into the 4 RVQ phases.
|
||||
|
||||
Phase ``p`` collects the tokenizer ids for codec ids
|
||||
``[p*1024, (p+1)*1024)``. Fails loudly on missing markers or an
|
||||
incomplete codec vocab.
|
||||
|
||||
Returns ``(phase_token_ids, audiogen_start_tid, audiogen_end_tid)``.
|
||||
"""
|
||||
vocab = tokenizer.get_vocab()
|
||||
start_tid = vocab.get("<audiogen_start>")
|
||||
end_tid = vocab.get("<audiogen_end>")
|
||||
if start_tid is None or end_tid is None:
|
||||
raise ValueError(f"Missing marker token ids: audiogen_start={start_tid}, audiogen_end={end_tid}")
|
||||
|
||||
phase_token_ids: list[list[int]] = []
|
||||
for phase in range(XCODEC1_NUM_CODEBOOKS):
|
||||
ids: list[int] = []
|
||||
missing: list[int] = []
|
||||
for codec_id in range(phase * XCODEC1_CODEBOOK_SIZE, (phase + 1) * XCODEC1_CODEBOOK_SIZE):
|
||||
tok = vocab.get(f"<audiocodec_{codec_id}>")
|
||||
if tok is None:
|
||||
missing.append(codec_id)
|
||||
else:
|
||||
ids.append(tok)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Incomplete codec vocab for RVQ phase {phase}: missing {len(missing)} codec ids (e.g. {missing[:5]})"
|
||||
)
|
||||
phase_token_ids.append(ids)
|
||||
|
||||
return phase_token_ids, int(start_tid), int(end_tid)
|
||||
|
||||
|
||||
def validate_rvq_phase(
|
||||
codec_ids: list[int],
|
||||
codebook_size: int = XCODEC1_CODEBOOK_SIZE,
|
||||
num_codebooks: int = XCODEC1_NUM_CODEBOOKS,
|
||||
) -> dict[str, Any]:
|
||||
"""Check that ``codec_ids[i]`` falls in RVQ phase ``i % 4``'s codec range."""
|
||||
mismatch_count = 0
|
||||
first_mismatch = None
|
||||
for i, codec_id in enumerate(codec_ids):
|
||||
expected = i % num_codebooks
|
||||
lo = expected * codebook_size
|
||||
if not (lo <= codec_id < lo + codebook_size):
|
||||
mismatch_count += 1
|
||||
if first_mismatch is None:
|
||||
first_mismatch = [i, int(codec_id), int(codec_id // codebook_size), int(expected)]
|
||||
return {
|
||||
"phase_valid": mismatch_count == 0,
|
||||
"mismatch_count": mismatch_count,
|
||||
"first_mismatch": first_mismatch,
|
||||
"codec_count": len(codec_ids),
|
||||
}
|
||||
|
||||
|
||||
class TTARVQPhaseMaskLogitsProcessor(LogitsProcessor):
|
||||
"""Mask logits to the RVQ-phase-valid token set for Audex TTA requests.
|
||||
|
||||
Per batch row:
|
||||
``_req[idx]`` -> {"start_tid", "end_tid", "codec_cap", "start_in_prompt"}
|
||||
``_output_tokens[idx]`` -> live reference to the request's output ids
|
||||
``_start_pos[idx]`` -> cached index of <audiogen_start> once seen;
|
||||
-1 means the prompt already ended with it.
|
||||
|
||||
The disallow masks are vocab-sized boolean tensors built once on the
|
||||
first ``apply`` (using the real ``logits.shape[-1]`` so vocab padding is
|
||||
also forbidden).
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device, is_pin_memory: bool) -> None:
|
||||
self.device = device
|
||||
self._req: dict[int, dict[str, Any]] = {}
|
||||
self._output_tokens: dict[int, list[int]] = {}
|
||||
self._start_pos: dict[int, int] = {}
|
||||
|
||||
self._phase_token_ids: list[list[int]] | None = None
|
||||
self._mask_end_tid: int | None = None
|
||||
|
||||
self._disallow_phase: torch.Tensor | None = None
|
||||
self._disallow_phase0_end: torch.Tensor | None = None
|
||||
self._disallow_except_end: torch.Tensor | None = None
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
return False
|
||||
|
||||
def _reset(self) -> None:
|
||||
self._req.clear()
|
||||
self._output_tokens.clear()
|
||||
self._start_pos.clear()
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None) -> None:
|
||||
if batch_update is None:
|
||||
return
|
||||
|
||||
for idx in batch_update.removed:
|
||||
self._req.pop(idx, None)
|
||||
self._output_tokens.pop(idx, None)
|
||||
self._start_pos.pop(idx, None)
|
||||
|
||||
if not self._req and batch_update.added:
|
||||
self._reset()
|
||||
|
||||
for idx, params, _, output_token_ids in batch_update.added:
|
||||
extra_args = params.extra_args if params else None
|
||||
tta = extra_args.get("tta_rvq") if extra_args else None
|
||||
if tta:
|
||||
self._req[idx] = {
|
||||
"start_tid": tta["start_tid"],
|
||||
"end_tid": tta["end_tid"],
|
||||
"codec_cap": tta.get("codec_cap"),
|
||||
"start_in_prompt": bool(tta.get("start_in_prompt", False)),
|
||||
}
|
||||
self._output_tokens[idx] = output_token_ids
|
||||
if self._req[idx]["start_in_prompt"]:
|
||||
self._start_pos[idx] = -1
|
||||
else:
|
||||
self._start_pos.pop(idx, None)
|
||||
if self._phase_token_ids is None:
|
||||
self._phase_token_ids = tta["phase_token_ids"]
|
||||
self._mask_end_tid = tta["end_tid"]
|
||||
else:
|
||||
self._req.pop(idx, None)
|
||||
self._output_tokens.pop(idx, None)
|
||||
self._start_pos.pop(idx, None)
|
||||
|
||||
if self._req:
|
||||
for src, dst, direction in batch_update.moved:
|
||||
src_req = self._req.pop(src, None)
|
||||
dst_req = self._req.pop(dst, None)
|
||||
src_tokens = self._output_tokens.pop(src, None)
|
||||
dst_tokens = self._output_tokens.pop(dst, None)
|
||||
src_pos = self._start_pos.pop(src, None)
|
||||
dst_pos = self._start_pos.pop(dst, None)
|
||||
if src_req is not None:
|
||||
self._req[dst] = src_req
|
||||
if src_tokens is not None:
|
||||
self._output_tokens[dst] = src_tokens
|
||||
if src_pos is not None:
|
||||
self._start_pos[dst] = src_pos
|
||||
if direction == MoveDirectionality.SWAP:
|
||||
if dst_req is not None:
|
||||
self._req[src] = dst_req
|
||||
if dst_tokens is not None:
|
||||
self._output_tokens[src] = dst_tokens
|
||||
if dst_pos is not None:
|
||||
self._start_pos[src] = dst_pos
|
||||
|
||||
def _build_masks(self, vocab_size: int) -> None:
|
||||
if self._phase_token_ids is None or self._mask_end_tid is None:
|
||||
raise RuntimeError(
|
||||
"TTARVQPhaseMaskLogitsProcessor: phase token ids missing; "
|
||||
"extra_args['tta_rvq'] was not provided for a TTA request."
|
||||
)
|
||||
if len(self._phase_token_ids) != XCODEC1_NUM_CODEBOOKS:
|
||||
raise RuntimeError(
|
||||
f"TTARVQPhaseMaskLogitsProcessor: expected {XCODEC1_NUM_CODEBOOKS} phase id "
|
||||
f"lists, got {len(self._phase_token_ids)}."
|
||||
)
|
||||
|
||||
allow = torch.zeros((XCODEC1_NUM_CODEBOOKS, vocab_size), dtype=torch.bool, device=self.device)
|
||||
for phase, ids in enumerate(self._phase_token_ids):
|
||||
if not ids:
|
||||
raise RuntimeError(f"TTARVQPhaseMaskLogitsProcessor: empty token id list for RVQ phase {phase}.")
|
||||
idx = torch.as_tensor(ids, dtype=torch.long, device=self.device)
|
||||
if int(idx.max()) >= vocab_size or int(idx.min()) < 0:
|
||||
raise RuntimeError(
|
||||
f"TTARVQPhaseMaskLogitsProcessor: phase {phase} token id out of range for vocab_size={vocab_size}."
|
||||
)
|
||||
allow[phase].index_fill_(0, idx, True)
|
||||
|
||||
end_onehot = torch.zeros(vocab_size, dtype=torch.bool, device=self.device)
|
||||
if self._mask_end_tid < 0 or self._mask_end_tid >= vocab_size:
|
||||
raise RuntimeError(
|
||||
f"TTARVQPhaseMaskLogitsProcessor: end token id out of range "
|
||||
f"for vocab_size={vocab_size}: {self._mask_end_tid}."
|
||||
)
|
||||
end_onehot[self._mask_end_tid] = True
|
||||
|
||||
self._disallow_phase = ~allow
|
||||
self._disallow_phase0_end = ~(allow[0] | end_onehot)
|
||||
self._disallow_except_end = ~end_onehot
|
||||
logger.info(
|
||||
"TTARVQPhaseMaskLogitsProcessor: built masks vocab=%d end_tid=%d (pid=%d)",
|
||||
vocab_size,
|
||||
self._mask_end_tid,
|
||||
os.getpid(),
|
||||
)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if not self._req:
|
||||
return logits
|
||||
|
||||
if self._disallow_phase is None:
|
||||
self._build_masks(logits.shape[-1])
|
||||
|
||||
num_rows = logits.shape[0]
|
||||
for idx, state in self._req.items():
|
||||
# Rows beyond this step's logits (pure-prefill / partially
|
||||
# scheduled steps) have nothing to mask; the persistent-batch
|
||||
# index re-enters range on the decode steps that sample.
|
||||
if idx >= num_rows:
|
||||
continue
|
||||
tokens = self._output_tokens.get(idx)
|
||||
if tokens is None:
|
||||
continue
|
||||
n = len(tokens)
|
||||
|
||||
start_pos = self._start_pos.get(idx)
|
||||
if start_pos is None:
|
||||
start_tid = state["start_tid"]
|
||||
for i in range(n):
|
||||
if tokens[i] == start_tid:
|
||||
start_pos = i
|
||||
self._start_pos[idx] = i
|
||||
break
|
||||
if start_pos is None:
|
||||
continue
|
||||
|
||||
position = n - start_pos - 1
|
||||
if position < 0:
|
||||
continue
|
||||
|
||||
cap = state["codec_cap"]
|
||||
if cap is not None and position >= cap:
|
||||
disallow = self._disallow_except_end
|
||||
else:
|
||||
phase = position & (XCODEC1_NUM_CODEBOOKS - 1)
|
||||
if phase == 0:
|
||||
disallow = self._disallow_phase0_end if position > 0 else self._disallow_phase[0]
|
||||
else:
|
||||
disallow = self._disallow_phase[phase]
|
||||
|
||||
logits[idx].masked_fill_(disallow, NEG_INF)
|
||||
|
||||
return logits
|
||||
@@ -84,6 +84,31 @@ _OMNI_MODELS = {
|
||||
"cosyvoice3",
|
||||
"CosyVoice3Model",
|
||||
),
|
||||
"NemotronDenseForCausalLM": (
|
||||
"audex",
|
||||
"audex_thinker",
|
||||
"NemotronDenseForCausalLM",
|
||||
),
|
||||
"AudexCode2Wav": (
|
||||
"audex",
|
||||
"audex_code2wav",
|
||||
"AudexCode2Wav",
|
||||
),
|
||||
"AudexXCodec1": (
|
||||
"audex",
|
||||
"audex_xcodec",
|
||||
"AudexXCodec1",
|
||||
),
|
||||
"NemotronDenseAudexForConditionalGeneration": (
|
||||
"audex",
|
||||
"audex_omni",
|
||||
"NemotronDenseAudexForConditionalGeneration",
|
||||
),
|
||||
"NemotronHAudexForConditionalGeneration": (
|
||||
"audex",
|
||||
"audex_omni",
|
||||
"NemotronHAudexForConditionalGeneration",
|
||||
),
|
||||
"OmniVoiceModel": (
|
||||
"omnivoice",
|
||||
"omnivoice",
|
||||
|
||||
Reference in new issue
Block a user