[New Model] support nemotron Audex (#4976)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-07-28 03:58:22 +00:00
1 parent f2c5152df0
commit 5014fea4a0
64 files changed
+7760 -39

No files matched your search

+5
View File
@@ -392,6 +392,11 @@ def test_sub_config_fields_match_rfc_scopes():
"enable_multithread_weight_load",
"num_weight_load_threads",
"disable_autocast",
# Per-stage checkpoint resolution for repos whose stages live in
# subfolders (e.g. Audex): mirrors StagePipelineConfig on the
# legacy engine-args path.
"model_subdir",
"tokenizer_subdir",
}
assert {f.name for f in fields(OmniStageLoadConfig)} == {
"load_format",
@@ -121,6 +121,10 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
("Qwen3TTSCode2Wav", "code2wav"),
("MiniCPMO45Code2Wav", "code2wav"),
("CosyVoice3Model", "cosyvoice3_code2wav"),
# audex: sync-path TTS decoder (async_chunk: false deploys) and
# the TTA XCodec1 stage (always sync full-payload).
("AudexCode2Wav", "audex_code2wav"),
("AudexXCodec1", "audex_xcodec"),
("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"),
("DyninOmniForConditionalGeneration", "token2image"),
("DyninOmniForConditionalGeneration", "token2audio"),
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HTTP + WebSocket validation for Qwen3-TTS Base and Higgs Audio V3 ``/v1/audio/speech`` (and related routes)."""
"""HTTP + WebSocket validation for Qwen3-TTS Base, Higgs Audio V3 and Audex TTA ``/v1/audio/speech`` (and related routes)."""
from __future__ import annotations
@@ -70,6 +70,19 @@ _HIGGS_AUDIO_V3_SPEECH = [
),
]
_AUDEX_TTA_CAPTION = "Heavy rain falling on a tin roof."
_AUDEX_TTA_SPEECH = [
pytest.param(
OmniServerParams(
model="nvidia/Nemotron-Labs-Audex-2B",
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
server_args=_SPEECH_SERVER_ARGS,
),
id="audex_tta",
marks=_L4_SPEECH_HW,
),
]
def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
if loc == "batch":
@@ -325,6 +338,37 @@ def test_speech_higgs_invalid_field_values(
)
# Audex TTA (``audex_tta`` deployment): the serving layer builds the caption
# prompt and injects the RVQ phase contract + default guidance server-side, so
# guidance knobs and speech-only fields must be rejected before generation.
@pytest.mark.parametrize(
"overrides, err_message",
[
pytest.param({"extra_params": {"cfg_scale": 99}}, "cfg_scale", id="cfg_scale_out_of_range"),
pytest.param({"extra_params": {"cfg_scale": "big"}}, "cfg_scale", id="cfg_scale_wrong_type"),
pytest.param({"extra_params": {"tta_rvq": {}}}, "tta_rvq", id="tta_rvq_not_overridable"),
pytest.param({"input": " "}, "caption", id="caption_whitespace_only"),
pytest.param({"voice": "alloy"}, "voice", id="voice_not_supported"),
pytest.param(
{"ref_audio": "https://example.com/ref.wav"},
"reference audio",
id="ref_audio_not_supported",
),
],
)
@pytest.mark.parametrize("omni_server", _AUDEX_TTA_SPEECH, indirect=True)
def test_speech_audex_tta_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
body = {"model": omni_server.model, "input": _AUDEX_TTA_CAPTION, **overrides}
openai_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
# ─────────────────────────────────────────────────────────────────────────────
# POST /v1/audio/speech/batch (JSON)
# ─────────────────────────────────────────────────────────────────────────────
@@ -0,0 +1,262 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline E2E test for the Audex cascaded speech-to-speech pipeline.
One ``audex_s2s`` deployment serves all three official
passes. The routing contract is the hard gate here: text-modality passes
(ASR, chat) finish at stage 0 and must never produce stage-1 audio; the
audio-modality TTS pass streams through the causal speech decoder.
"""
from __future__ import annotations
import copy
import os
import re
import time
import numpy as np
import pytest
from vllm.assets.audio import AudioAsset
from tests.helpers.mark import hardware_test
from tests.helpers.media import concat_audio
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
SAMPLE_RATE = 16_000
ASR_PROMPT = (
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
)
# Official chat-pass recipe (examples/offline_inference/audex/speech_to_speech.py,
# mirroring the official cascaded web demo) — 30B-A3B ONLY. The
# formatting-instruction prefix is load-bearing on the 30B-A3B: without it a bare
# transcript turn is treated as a TTS request and the model emits literal
# <speechcodec_N> tokens instead of text, and under the yaml's near-greedy
# defaults its reasoning block never closes. The 2B must NOT use this recipe:
# it is long-verified stable on a bare prompt + engine-default sampling, while
# the official temp-1.0 seed-0 draw hallucinates off-language text (yet closes
# </think> cleanly, so the seed retry never fires) and fails the round-trip
# WER gate — measured, reproduced byte-identically.
SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
TEXT_INSTRUCTION = (
"SYSTEM & FORMATTING INSTRUCTION: You are Audex, created by NVIDIA based on the Nemotron-Cascade-2 architecture. "
"You may output your reasoning, followed by your final response. "
"You may format your reasoning block however you like. "
"However, your final response must strictly follow these rules: "
"* Write in plain, unformatted prose like a book or newspaper article. "
"* Do not use markdown, bullet points, lists, or headers. "
"* Standard numbers, abbreviations, and symbols are acceptable. "
"* [CRITICAL] You must press enter after every single sentence, placing each sentence on its own separate line."
)
CHAT_MAX_TOKENS = 2048 # official --text-max-tokens default
# The official demo samples this pass unseeded; some trajectories deliberate
# past the token budget without ever closing the think block, so retry a few
# deterministic seeds until one closes.
CHAT_SEEDS = (0, 1, 2)
_audex_deployment = get_deploy_config_path("audex_s2s.yaml")
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
_OMNI_RUNNER_PARAMS = [
pytest.param(
(_audex_model, _audex_deployment, {"async_chunk": True}),
id="s2s_full",
),
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
_OMNI_RUNNER_PARAMS.append(
pytest.param(
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_s2s_30b.yaml"), {"async_chunk": True}),
id="s2s_30b",
)
)
pytestmark = [
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
]
def _normalize(text: str) -> list[str]:
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
def _wer(hyp: str, ref: str) -> float:
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
if not ref_words:
return 0.0
rows = list(range(len(hyp_words) + 1))
for i, ref_word in enumerate(ref_words, 1):
prev, rows[0] = rows[0], i
for j, hyp_word in enumerate(hyp_words, 1):
cur = min(rows[j] + 1, rows[j - 1] + 1, prev + (ref_word != hyp_word))
prev, rows[j] = rows[j], cur
return rows[-1] / len(ref_words)
def _chat_prompt(user_text: str) -> str:
# Official chat-pass recipe: system prompt + formatting instruction
# prefixed to the user text, thinking ENABLED via an open <think> block
# (chat_template.jinja with enable_thinking=True renders
# "<|im_start|>assistant\n<think>\n"). Priming the block open keeps the
# reasoning inside <think>...</think> so _extract_final_response can drop
# it; with a closed block the 30B leaks the reasoning as plain text.
return (
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
f"<|im_start|>user\n{TEXT_INSTRUCTION}\n\n{user_text}<|im_end|>\n"
"<|im_start|>assistant\n<think>\n"
)
def _extract_final_response(text: str) -> str:
# Mirror the official demo's extract_spoken_answer: keep only what
# follows the LAST closing think tag, falling back to the full text when
# the model skipped reasoning or never closed the block. Benign no-op for
# the 2B, whose answers carry no think block.
if "</think>" in text:
answer = text.rsplit("</think>", 1)[-1].strip()
if answer:
return answer
return text.strip()
def _chat_params(runner: OmniRunner, seed: int):
# Official demo text-pass sampling (--text-temperature 1.0, --text-top-p
# 0.95, --text-max-tokens 2048). The yaml stage-0 defaults (temperature
# 0.1, top_k 80) are TTS-thinker settings; under near-greedy decoding the
# 30B loops inside its reasoning block until the token budget runs out
# and never emits </think>.
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
stage0 = params[0]
stage0.temperature = 1.0
stage0.top_p = 0.95
stage0.top_k = -1
stage0.max_tokens = CHAT_MAX_TOKENS
stage0.seed = seed
return params
def _chat_once(runner: OmniRunner, prompt: str, seed: int) -> tuple[str, bool]:
(req_output,) = runner.omni.generate([{"prompt": prompt, "modalities": ["text"]}], _chat_params(runner, seed))
out = req_output.outputs[0]
text = (out.text or "").strip()
truncated = getattr(out, "finish_reason", None) == "length" or len(out.token_ids or []) >= CHAT_MAX_TOKENS
return text, truncated
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_audex_s2s_cascade_round_trip(omni_runner: OmniRunner, run_level: str) -> None:
"""ASR → chat → TTS over one deployment; text passes never emit audio."""
real_weights = run_level in {"advanced_model", "full_model"}
# Spoken question: vLLM's public audio asset (downloaded on first use).
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
# Pass 1 — ASR (text-final).
asr_outputs = omni_runner.omni.generate(
[
{
"prompt": ASR_PROMPT,
"multi_modal_data": {"audio": (audio, sr)},
"modalities": ["text"],
}
]
)
assert len(asr_outputs) == 1
asr_out = asr_outputs[0]
transcript = (asr_out.outputs[0].text or "").strip()
mm = getattr(asr_out, "multimodal_output", None) or {}
stage1_audio = concat_audio(mm.get("audio")) if "audio" in mm else np.zeros((0,), dtype=np.float32)
assert stage1_audio.size == 0, "Text-modality ASR pass produced stage-1 audio (routing regression)"
# Pass 2 — chat (text-final). The recipe is size-conditional (see the
# SYSTEM_PROMPT comment above): the 30B requires the official recipe,
# the 2B regresses under it.
if omni_runner.model_name == _AUDEX_30B_MODEL:
# Official recipe. The raw completion is
# <reasoning...></think><answer>; speak only the final response.
# Retry with the next seed only when the budget ran out before the
# reasoning block closed (a natural stop without </think> is a
# complete answer); if every seed spins, keep the full text
# (official demo behavior).
chat_prompt = _chat_prompt(transcript or "Hello.")
raw = ""
for seed in CHAT_SEEDS:
raw, truncated = _chat_once(omni_runner, chat_prompt, seed)
if "</think>" in raw or not truncated:
break
answer = _extract_final_response(raw)
else:
# 2B: bare prompt with a closed think block, engine-default sampling
# (yaml near-greedy), no retry; the answer is used as-is.
chat_prompt = f"<|im_start|>user\n{transcript or 'Hello.'}<|im_end|>\n<|im_start|>assistant\n<think></think>"
(chat_output,) = omni_runner.omni.generate([{"prompt": chat_prompt, "modalities": ["text"]}])
answer = (chat_output.outputs[0].text or "").strip()
if real_weights:
assert transcript, "ASR pass produced an empty transcript"
assert answer, "Chat pass produced an empty answer"
# Pass 3 — TTS (audio-final; streams through code2wav). Both final
# stages may emit an output record; collect the audio-bearing one.
# The formatting instruction puts every sentence on its own line;
# flatten to plain prose for the TTS thinker.
tts_text = re.sub(r"\s+", " ", answer).strip() if answer else "The weather is nice today."
tts_text = tts_text[:200]
tts_prompt = {"prompt": build_cond_prompt(tts_text), "modalities": ["audio"]}
tts_outputs = omni_runner.omni.generate([tts_prompt])
assert tts_outputs, "TTS pass returned no outputs"
speech = np.zeros((0,), dtype=np.float32)
for req_output in tts_outputs:
mm = getattr(req_output, "multimodal_output", None) or {}
if "audio" in mm:
speech = concat_audio(mm["audio"])
break
assert speech.size > 0, "TTS pass produced empty audio"
if real_weights:
rms = float(np.sqrt(np.mean(np.square(speech))))
assert rms > 1e-3, f"TTS-pass audio near-silent (rms={rms})"
# Pass 4 — output verification: transcribe the synthesized answer
# with the SAME deployment's ASR pass and compare to the answer
# text (self-contained; no external ASR dependency).
verify_outputs = omni_runner.omni.generate(
[
{
"prompt": ASR_PROMPT,
"multi_modal_data": {"audio": (speech, SAMPLE_RATE)},
"modalities": ["text"],
}
]
)
verify_text = (verify_outputs[0].outputs[0].text or "").strip()
assert verify_text, "Output-verification ASR pass returned empty text"
# The full checkpoint prefixes transcripts with a language-tag
# sentence and quotes the content; extract the quoted payload.
if "'" in verify_text and len(verify_text.split("'")) >= 3:
verify_text = verify_text.split("'")[1].strip()
wer = _wer(verify_text, tts_text)
# Hard gate: the synthesized WAV must transcribe back to the answer
# (lenient threshold absorbs ASR quoting/prefix differences).
assert wer <= 0.5, f"Output WAV does not match the answer (WER={wer:.2f}): {verify_text!r} vs {tts_text!r}"
print(f"[trend] Audex S2S output-ASR WER vs answer: {wer:.3f} ({verify_text!r})")
# Final pass — time-to-first-audio for the streaming TTS path. This
# MUST be the last engine call: the py_generator API closes the engine
# when the generator is exhausted.
t_start = time.perf_counter()
first_audio_s: float | None = None
for req_output in omni_runner.omni.generate([tts_prompt], py_generator=True):
mm = getattr(req_output, "multimodal_output", None) or {}
if "audio" in mm and concat_audio(mm["audio"]).size > 0:
if first_audio_s is None:
first_audio_s = time.perf_counter() - t_start
assert first_audio_s is not None, "streaming TTS never produced an audio-bearing chunk"
print(f"[trend] Audex S2S TTS-pass first-audio latency: {first_audio_s * 1000.0:.0f} ms")
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline E2E smoke test for Audex audio understanding (thinker-only).
Single-stage ``audex_thinker_only`` pipeline on
``checkpoint_folder_full``: WAV (+ instruction) in, text out. The input clip
is vLLM's public ``mary_had_lamb`` audio asset (downloaded on first use), so
its transcript is known.
WER is a trend metric (recorded in the assertion message, no numeric hard
gate); the hard gates are structural: non-empty, non-degenerate text.
"""
from __future__ import annotations
import os
import pytest
from vllm.assets.audio import AudioAsset
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
# Full transcript of the mary_had_lamb asset (Edison's phonograph recording);
# a shortened reference would count the extra spoken words as insertions and
# inflate WER past the trend threshold.
_REFERENCE = (
"the first words i spoke in the original phonograph a little piece of practical poetry "
"mary had a little lamb its fleece was white as snow and everywhere that mary went the "
"lamb was sure to go"
)
ASR_PROMPT = (
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
)
_audex_deployment = get_deploy_config_path("audex_thinker_only.yaml")
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
_OMNI_RUNNER_PARAMS = [
pytest.param(
(_audex_model, _audex_deployment, {"async_chunk": False}),
id="thinker_only",
),
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
_OMNI_RUNNER_PARAMS.append(
pytest.param(
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_thinker_only_30b.yaml"), {"async_chunk": False}),
id="thinker_only_30b",
)
)
pytestmark = [
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
]
def _normalize(text: str) -> list[str]:
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
def _wer(hyp: str, ref: str) -> float:
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
if not ref_words:
return 0.0
rows = list(range(len(hyp_words) + 1))
for i, ref_word in enumerate(ref_words, 1):
prev, rows[0] = rows[0], i
for j, hyp_word in enumerate(hyp_words, 1):
cur = min(
rows[j] + 1,
rows[j - 1] + 1,
prev + (ref_word != hyp_word),
)
prev, rows[j] = rows[j], cur
return rows[-1] / len(ref_words)
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_audex_offline_asr_smoke(omni_runner: OmniRunner, run_level: str) -> None:
"""Transcribing the public audio asset yields coherent, non-degenerate text."""
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
outputs = omni_runner.omni.generate([{"prompt": ASR_PROMPT, "multi_modal_data": {"audio": (audio, sr)}}])
assert len(outputs) == 1
text = outputs[0].outputs[0].text if outputs[0].outputs else ""
assert isinstance(text, str)
if run_level in {"advanced_model", "full_model"}:
words = _normalize(text)
assert words, f"Empty transcription: {text!r}"
# Degenerate repetition guard: no word may dominate the output.
most_common = max(words.count(w) for w in set(words))
assert most_common <= max(3, len(words) // 2), f"Degenerate transcript: {text!r}"
wer = _wer(text, _REFERENCE)
# Trend metric (recorded, not gated): substantially-correct check only.
assert wer <= 0.9, f"Transcript unrelated to reference (WER={wer:.2f}): {text!r}"
print(f"[trend] Audex S2T WER vs reference: {wer:.3f} ({text!r})")
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline E2E smoke test for Audex text-to-audio (TTA).
Caption → RVQ-phase-masked <audiocodec_*> tokens → XCodec1 waveform through
the ``audex_tta`` pipeline. RVQ phase validity is enforced by
the mask logits processor (unit-tested against the official validator); the
hard gates here are decode-side: non-silent, finite audio at the documented
sample rate. CFG runs at the official TTA default (3.0).
"""
from __future__ import annotations
import copy
import os
import numpy as np
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import concat_audio
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
SAMPLE_RATE = 16_000
# The acceptance gate requires eight captions decoding to non-silent,
# finite audio through XCodec1.
CAPTIONS = (
"Heavy rain falling on a tin roof.",
"A dog barking in the distance while birds chirp.",
"Ocean waves crashing on a rocky shore.",
"A crackling campfire at night with crickets.",
"Thunder rumbling during a storm.",
"A steam train whistle blowing as it departs.",
"Wind chimes ringing in a gentle breeze.",
"Footsteps crunching through fresh snow.",
)
_audex_deployment = get_deploy_config_path("audex_tta.yaml")
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
_OMNI_RUNNER_PARAMS = [
pytest.param(
(_audex_model, _audex_deployment, {"async_chunk": False}),
id="tta_full_payload",
),
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
_OMNI_RUNNER_PARAMS.append(
pytest.param(
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tta_30b.yaml"), {"async_chunk": False}),
id="tta_30b",
)
)
pytestmark = [
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
]
@pytest.fixture(scope="module")
def tta_tokenizer_bits():
from transformers import AutoTokenizer
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids
root = ensure_audex_snapshot(_audex_model, profile="tta")
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer)
return tokenizer, phase_token_ids, start_tid, end_tid
def _tta_sampling_params(runner: OmniRunner, cond_prompt: str, tokenizer_bits, cfg_scale: float = 3.0):
from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt
tokenizer, phase_token_ids, start_tid, end_tid = tokenizer_bits
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
stage0 = params[0]
if stage0.extra_args is None:
stage0.extra_args = {}
stage0.extra_args["tta_rvq"] = {
"phase_token_ids": phase_token_ids,
"start_tid": start_tid,
"end_tid": end_tid,
"codec_cap": 4000,
"start_in_prompt": True,
}
if cfg_scale > 1.0:
stage0.extra_args.update(
{
"cfg_scale": float(cfg_scale),
"cfg_role": "cond",
"cfg_pair_id": f"e2e-tta-{abs(hash(cond_prompt)) % 10_000_000}",
"cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer),
}
)
return params
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_audex_offline_tta_eight_captions(omni_runner: OmniRunner, run_level: str, tta_tokenizer_bits) -> None:
"""All eight captions decode through XCodec1 to non-silent, finite audio.
RVQ phase validity is enforced by the mask logits processor by
construction; the final user output carries only decoded audio (the
audio-final pipeline emits no stage-0 token record), and disabling the
mask in-engine as an e2e control would require a second engine boot
with a different logits-processor config. The mask's sensitivity is
therefore proven at unit level (see test_audex_tta.py's mask tests and
the unmasked-sampling control) against the official validator.
"""
real_weights = run_level in {"advanced_model", "full_model"}
for caption in CAPTIONS:
prompt = build_tta_cond_prompt(caption)
params = _tta_sampling_params(omni_runner, prompt, tta_tokenizer_bits)
outputs = omni_runner.omni.generate([prompt], params)
assert len(outputs) == 1, f"{caption!r}: expected one output, got {len(outputs)}"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, f"{caption!r}: no audio output: {list(audio_mm.keys())}"
audio = concat_audio(audio_mm["audio"])
assert audio.size > 0, f"{caption!r}: generated audio is empty"
assert np.isfinite(audio).all(), f"{caption!r}: audio contains NaN/Inf"
sr_val = audio_mm.get("sr", SAMPLE_RATE)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
assert int(sr_val) == SAMPLE_RATE, f"{caption!r}: unexpected sample_rate={sr_val}"
if real_weights:
duration_s = audio.size / SAMPLE_RATE
# The decode cap is 500 frames = 10 s at 50 fps.
assert 0.1 <= duration_s <= 10.5, f"{caption!r}: unexpected duration={duration_s:.3f}s"
rms = float(np.sqrt(np.mean(np.square(audio))))
assert rms > 1e-4, f"{caption!r}: audio is near-silent (rms={rms})"
@@ -0,0 +1,161 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline E2E smoke test for Audex (Nemotron-Labs-Audex-2B) TTS.
Passes the HF repo ROOT and verifies the 2-stage pipeline (thinker →
streaming causal speech decoder): root-path/subfolder resolution, codec-token
extraction, per-request streaming sessions, and the lookahead flush at EOS.
"""
from __future__ import annotations
import os
import numpy as np
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import concat_audio
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
SAMPLE_RATE = 16_000
SYNTH_TEXTS = (
"The weather is so good, and I want to enjoy the beautiful morning in the park.",
"The quick brown fox jumps over the lazy dog.",
)
_audex_deployment = get_deploy_config_path("audex_tts.yaml")
# Collection must not download anything: pass the repo id (or a local
# override) and let the engine's stage-init path resolve/download the
# required snapshot subset at execution time (ensure_audex_snapshot).
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
_OMNI_RUNNER_PARAMS = [
pytest.param(
(_audex_model, _audex_deployment, {"async_chunk": True}),
id="async_chunk",
),
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
_OMNI_RUNNER_PARAMS.append(
pytest.param(
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tts_30b.yaml"), {"async_chunk": True}),
id="tts_30b",
)
)
pytestmark = [
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
]
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_audex_offline_tts_smoke(omni_runner: OmniRunner, run_level: str) -> None:
"""Audex TTS from the repo root should produce sane 16 kHz audio per prompt.
At ``core_model`` level the runner loads DUMMY weights (structural smoke
only); real-speech assertions apply from ``advanced_model`` up.
"""
prompts = [build_cond_prompt(text) for text in SYNTH_TEXTS]
outputs = omni_runner.omni.generate(prompts)
real_weights = run_level in {"advanced_model", "full_model"}
assert len(outputs) == len(SYNTH_TEXTS), f"expected {len(SYNTH_TEXTS)} outputs, got {len(outputs)}"
for output in outputs:
audio_mm = output.multimodal_output
assert "audio" in audio_mm, f"No audio output found: {list(audio_mm.keys())}"
audio = concat_audio(audio_mm["audio"])
assert audio.size > 0, "Generated audio is empty"
sr_val = audio_mm.get("sr", SAMPLE_RATE)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
assert int(sr_val) == SAMPLE_RATE, f"Unexpected sample_rate={sr_val}"
if real_weights:
duration_s = audio.size / SAMPLE_RATE
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
# Real speech, not silence.
rms = float(np.sqrt(np.mean(np.square(audio))))
assert rms > 1e-3, f"Audio is near-silent (rms={rms})"
def _cfg_sampling_params(runner: OmniRunner, cfg_scale: float, pair_id: str, cond_prompt: str):
"""Stage sampling params carrying the CFG pair contract for one request."""
import copy
from transformers import AutoTokenizer
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
from vllm_omni.model_executor.models.audex.prompt import build_null_prompt
# Resolve the ACTIVE param's model (2B or 30B), not the module default:
# the null prompt must be built with the same tokenizer the engine uses.
root = ensure_audex_snapshot(runner.model_name)
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
stage0 = params[0]
if stage0.extra_args is None:
stage0.extra_args = {}
stage0.extra_args.update(
{
"cfg_scale": float(cfg_scale),
"cfg_role": "cond",
"cfg_pair_id": pair_id,
"cfg_null_prompt": build_null_prompt(cond_prompt, tokenizer),
}
)
return params
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_audex_offline_cfg_guided_single_stream(omni_runner: OmniRunner, run_level: str) -> None:
"""A guided request must yield exactly ONE audio stream (companion suppressed)."""
prompt = build_cond_prompt(SYNTH_TEXTS[0])
params = _cfg_sampling_params(omni_runner, 1.5, "e2e-cfg-pair", prompt)
outputs = omni_runner.omni.generate([prompt], params)
assert len(outputs) == 1, f"expected exactly one output (no companion leak), got {len(outputs)}"
audio = concat_audio(outputs[0].multimodal_output["audio"])
assert audio.size > 0, "Guided generation produced empty audio"
if run_level in {"advanced_model", "full_model"}:
rms = float(np.sqrt(np.mean(np.square(audio))))
assert rms > 1e-3, f"Guided audio is near-silent (rms={rms})"
duration_s = audio.size / SAMPLE_RATE
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_audex_offline_cfg_disabled_is_byte_identical(omni_runner: OmniRunner, run_level: str) -> None:
"""cfg_scale=1.0 must be indistinguishable from not passing cfg at all."""
prompt = build_cond_prompt(SYNTH_TEXTS[1])
baseline = omni_runner.omni.generate([prompt])
base_audio = concat_audio(baseline[0].multimodal_output["audio"])
import copy
params = copy.deepcopy(omni_runner.omni.resolve_sampling_params_list(None))
if params[0].extra_args is None:
params[0].extra_args = {}
params[0].extra_args["cfg_scale"] = 1.0
disabled = omni_runner.omni.generate([prompt], params)
disabled_audio = concat_audio(disabled[0].multimodal_output["audio"])
assert base_audio.shape == disabled_audio.shape, (
f"cfg_scale=1.0 changed output length: {base_audio.shape} vs {disabled_audio.shape}"
)
np.testing.assert_array_equal(base_audio, disabled_audio)
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) text-to-audio.
Covers /v1/audio/speech through the ``audex_tta`` deployment:
the ``audex_tta`` adapter builds the caption prompt and the serving layer
injects the RVQ phase contract plus the official default guidance
(cfg_scale 3.0) server-side.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.slow, pytest.mark.tts]
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
CAPTION = "Heavy rain falling on a tin roof."
SAMPLE_RATE = 16_000
audex_tta_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
server_args=["--trust-remote-code"],
),
id="audex_tta",
)
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
audex_tta_server_params.append(
pytest.param(
OmniServerParams(
model=_AUDEX_30B_MODEL,
stage_config_path=get_deploy_config_path("audex_tta_30b.yaml"),
server_args=["--trust-remote-code"],
),
id="online_tta_30b",
)
)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_tta_server_params, indirect=True)
def test_audex_tta_online_smoke(omni_server, openai_client, run_level) -> None:
"""A caption returns a valid finite non-silent WAV at the default cfg 3.0."""
import io
import wave
import numpy as np
import requests
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
resp = requests.post(
url,
json={"model": omni_server.model, "input": CAPTION, "response_format": "wav"},
timeout=600,
)
assert resp.status_code == 200, resp.text[:200]
assert resp.content[:4] == b"RIFF", "not a WAV response"
with wave.open(io.BytesIO(resp.content)) as wav:
assert wav.getframerate() == SAMPLE_RATE, f"unexpected sample rate {wav.getframerate()}"
frames = wav.readframes(wav.getnframes())
duration_s = wav.getnframes() / wav.getframerate()
audio = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
assert audio.size > 0, "empty audio body"
assert np.isfinite(audio).all(), "audio contains NaN/Inf"
if run_level in {"advanced_model", "full_model"}:
# The decode cap is 500 frames = 10 s at 50 fps.
assert 0.1 <= duration_s <= 10.5, f"unexpected duration={duration_s:.3f}s"
rms = float(np.sqrt(np.mean(np.square(audio))))
assert rms > 1e-4, f"audio is near-silent (rms={rms})"
+230
View File
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) TTS.
Covers /v1/audio/speech non-streaming and audio-byte streaming, plus the
Audex request policies: single built-in voice (missing/empty/"default" only),
no reference audio, and cfg_scale reserved at 1.0.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.slow, pytest.mark.tts]
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
PROMPT = "The weather is so good, and I want to enjoy the beautiful morning in the park."
audex_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_deploy_config_path("audex_tts.yaml"),
server_args=["--trust-remote-code"],
),
id="audex",
)
]
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
if os.environ.get("AUDEX_E2E_30B") == "1":
audex_server_params.append(
pytest.param(
OmniServerParams(
model=_AUDEX_30B_MODEL,
stage_config_path=get_deploy_config_path("audex_tts_30b.yaml"),
server_args=["--trust-remote-code"],
),
id="online_tts_30b",
)
)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_nonstream(omni_server, openai_client) -> None:
"""Plain English TTS, non-streaming WAV response."""
request_config = {
"model": omni_server.model,
"input": PROMPT,
"stream": False,
"response_format": "wav",
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_stream_audio(omni_server, openai_client) -> None:
"""Plain English TTS, raw audio-byte streaming."""
request_config = {
"model": omni_server.model,
"input": PROMPT,
"stream": True,
"stream_format": "audio",
"response_format": "wav",
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_max_new_tokens_caps_duration(omni_server, openai_client, run_level) -> None:
"""max_new_tokens must cap stage-0 generation (review P2 regression)."""
import io
import wave
import requests
if run_level not in {"advanced_model", "full_model"}:
pytest.skip("duration cap needs real weights")
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
resp = requests.post(
url,
json={
"model": omni_server.model,
"input": PROMPT,
"response_format": "wav",
"max_new_tokens": 25,
},
)
assert resp.status_code == 200, resp.text
with wave.open(io.BytesIO(resp.content)) as wav:
duration_s = wav.getnframes() / wav.getframerate()
# 25 codec frames at 50 fps = 0.5 s (+ small lookahead tail); the full
# sentence would be ~3 s, so a capped result proves the override applied.
assert duration_s <= 1.0, f"max_new_tokens ignored: duration={duration_s:.2f}s"
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_request_policies(omni_server, openai_client) -> None:
"""Unsupported voice / CFG / empty input must be rejected with 400."""
import requests
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
ok = requests.post(url, json={"model": omni_server.model, "input": PROMPT, "voice": "default"})
assert ok.status_code == 200, ok.text
for bad_payload, needle in (
({"input": PROMPT, "voice": "alloy"}, "voice"),
({"input": " "}, "non-empty"),
({"input": PROMPT, "extra_params": {"cfg_scale": 0.5}}, "cfg_scale"),
({"input": PROMPT, "extra_params": {"cfg_scale": 10.5}}, "cfg_scale"),
({"input": PROMPT, "extra_params": {"cfg_scale": "big"}}, "cfg_scale"),
({"input": PROMPT, "extra_params": {"cfg_role": "cond"}}, "cfg_role"),
({"input": PROMPT, "extra_params": {"cfg_pair_id": "x"}}, "cfg_pair_id"),
({"input": PROMPT, "ref_audio": "https://example.com/ref.wav"}, "reference audio"),
):
resp = requests.post(url, json={"model": omni_server.model, **bad_payload})
assert resp.status_code == 400, f"{bad_payload} -> {resp.status_code}: {resp.text[:200]}"
assert needle in resp.text, f"{bad_payload} error missing {needle!r}: {resp.text[:200]}"
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_cfg_nonstream(omni_server, openai_client) -> None:
"""Guided (cfg_scale=1.5) non-streaming request returns one WAV."""
request_config = {
"model": omni_server.model,
"input": PROMPT,
"stream": False,
"response_format": "wav",
"extra_params": {"cfg_scale": 1.5},
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_cfg_stream_audio(omni_server, openai_client) -> None:
"""Guided (cfg_scale=1.5) streaming request yields a single audio stream."""
request_config = {
"model": omni_server.model,
"input": PROMPT,
"stream": True,
"stream_format": "audio",
"response_format": "wav",
"extra_params": {"cfg_scale": 1.5},
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_mixed_cfg_concurrency(omni_server, openai_client) -> None:
"""Mixed guided/unguided requests at concurrency 4 all complete with audio."""
import concurrent.futures
import requests
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
payloads = [
{"model": omni_server.model, "input": f"{PROMPT} Take {i}.", "response_format": "wav"}
| ({"extra_params": {"cfg_scale": 1.5}} if i % 2 == 0 else {})
for i in range(4)
]
def _post(payload):
return requests.post(url, json=payload, timeout=300)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
responses = list(pool.map(_post, payloads))
for payload, resp in zip(payloads, responses):
assert resp.status_code == 200, f"{payload} -> {resp.status_code}: {resp.text[:200]}"
assert resp.content[:4] == b"RIFF", f"{payload}: not a WAV response"
assert len(resp.content) > 44, f"{payload}: empty audio body"
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
def test_audex_tts_stream_abort_leaves_server_healthy(omni_server, openai_client) -> None:
"""Aborting a guided streaming request mid-flight must not wedge the server.
The abort cascades to the CFG companion (cfg_companion_tracker); a
follow-up guided request on the same server must still succeed.
"""
import requests
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
long_input = " ".join([PROMPT] * 4)
with requests.post(
url,
json={
"model": omni_server.model,
"input": long_input,
"stream": True,
"stream_format": "audio",
"response_format": "wav",
"extra_params": {"cfg_scale": 1.5},
},
stream=True,
timeout=300,
) as resp:
assert resp.status_code == 200, resp.text
# Read one chunk, then abort the connection mid-stream.
next(resp.iter_content(chunk_size=4096))
follow_up = requests.post(
url,
json={
"model": omni_server.model,
"input": PROMPT,
"response_format": "wav",
"extra_params": {"cfg_scale": 1.5},
},
timeout=300,
)
assert follow_up.status_code == 200, follow_up.text[:200]
assert follow_up.content[:4] == b"RIFF" and len(follow_up.content) > 44
@@ -0,0 +1,376 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Serving-layer regressions for the Audex zero-codec-token contract.
A zero-codec-token (or phase-invalid TTA) request reaches the serving layer
as a stream whose only audio payloads are zero-size tensors. The streaming
generator must not turn that into a header-only/empty successful stream: the
raw-bytes generator raises (before any WAV header), and the SSE wrapper
surfaces ``speech.audio.error`` instead of ``speech.audio.done``. The
contract covers both audex pipelines (``audex`` TTS and ``audex_tta``).
"""
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.entrypoints.openai.serving_speech import (
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES,
OmniOpenAIServingSpeech,
)
def _serving(model_type: str = "audex") -> OmniOpenAIServingSpeech:
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
serving._tts_model_type = model_type
serving._mark_ref_audio_artifact_ready_for_request = lambda request_id: None
serving._discard_ref_audio_artifact_warmup = lambda request_id: None
return serving
def _result(audio: torch.Tensor) -> SimpleNamespace:
return SimpleNamespace(multimodal_output={"model_outputs": audio, "sr": torch.tensor(16000, dtype=torch.int32)})
async def _gen(*results):
for res in results:
yield res
async def _collect(chunks):
out = []
async for chunk in chunks:
out.append(chunk)
return out
def test_no_audio_guard_covers_both_audex_pipelines():
# The non-streaming CreateAudio guard keys off the same constant; pin
# its membership so neither pipeline regresses to empty-200 responses.
assert _AUDEX_NO_AUDIO_GUARD_MODEL_TYPES == {"audex", "audex_tta"}
@pytest.mark.asyncio
@pytest.mark.parametrize("model_type", ["audex", "audex_tta"])
async def test_raw_stream_raises_on_empty_audio_before_any_header(model_type):
serving = _serving(model_type)
chunks = serving._generate_audio_chunks(
_gen(_result(torch.empty(0)), _result(torch.empty(0))),
request_id="req-empty",
response_format="wav",
)
with pytest.raises(ValueError, match="no audio"):
await _collect(chunks)
@pytest.mark.asyncio
async def test_raw_stream_yields_audio_normally_when_non_empty():
serving = _serving()
chunks = serving._generate_audio_chunks(
_gen(_result(torch.zeros(320)), _result(torch.empty(0))),
request_id="req-ok",
response_format="wav",
)
collected = await _collect(chunks)
# WAV header + one PCM chunk; the trailing empty chunk is skipped silently.
assert len(collected) == 2
assert collected[0][:4] == b"RIFF"
assert len(collected[1]) == 320 * 2 # int16 PCM
@pytest.mark.asyncio
@pytest.mark.parametrize("model_type", ["audex", "audex_tta"])
async def test_sse_stream_emits_error_not_done_on_empty_audio(model_type):
serving = _serving(model_type)
events = serving._generate_audio_sse_events(
_gen(_result(torch.empty(0))),
request_id="req-empty",
response_format="pcm",
)
collected = await _collect(events)
assert any("speech.audio.error" in event for event in collected)
assert not any("speech.audio.done" in event for event in collected)
@pytest.mark.asyncio
async def test_non_audex_empty_chunks_keep_legacy_behavior():
"""Other models keep their existing semantics (no new rejection)."""
serving = _serving(model_type="qwen3_tts")
chunks = serving._generate_audio_chunks(
_gen(_result(torch.empty(0))),
request_id="req-other",
response_format="pcm",
)
await _collect(chunks) # must not raise
# ---------------------------------------------------------------- CFG serving surface
import re # noqa: E402
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest # noqa: E402
from vllm_omni.entrypoints.openai.tts_adapters.audex import AudexAdapter # noqa: E402
def _adapter() -> AudexAdapter:
return AudexAdapter.__new__(AudexAdapter)
def _speech_request(**extra_params) -> OpenAICreateSpeechRequest:
payload = {"model": "audex", "input": "Hello world."}
if extra_params:
payload["extra_params"] = extra_params
return OpenAICreateSpeechRequest(**payload)
class _MarkerTokenizer:
"""Special markers count one token; words and punctuation split."""
_TOKEN_RE = re.compile(r"<unk>|<\|[^|]*\|>|<[a-z_]+>|\w+|[^\w\s]")
def encode(self, text: str) -> list[int]:
return [0] * len(self._TOKEN_RE.findall(text))
class TestAudexCfgValidation:
def test_no_cfg_scale_accepted(self):
assert _adapter().validate(_speech_request()) is None
@pytest.mark.parametrize("scale", [1.0, 1.5, 3.0, 10.0])
def test_in_range_scale_accepted(self, scale):
assert _adapter().validate(_speech_request(cfg_scale=scale)) is None
@pytest.mark.parametrize("scale", [0.5, 0.0, -1.0, 10.5, "big"])
def test_out_of_range_or_malformed_scale_rejected(self, scale):
error = _adapter().validate(_speech_request(cfg_scale=scale))
assert error is not None and "cfg_scale" in error
def test_explicit_null_scale_treated_as_absent(self):
assert _adapter().validate(_speech_request(cfg_scale=None)) is None
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
params = SimpleNamespace(extra_args={"cfg_scale": None})
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params)
assert out.extra_args == {}
@pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt"])
def test_internal_cfg_keys_rejected(self, key):
error = _adapter().validate(_speech_request(**{key: "x"}))
assert error is not None and key in error
class TestAudexCfgInjection:
def _serving_with_fake_tokenizer(self):
serving = _serving()
serving._get_audex_tokenizer = lambda: _MarkerTokenizer()
return serving
def _cond_prompt(self, text: str = "Hello world.") -> str:
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
return build_cond_prompt(text)
def test_no_extra_args_is_untouched(self):
serving = self._serving_with_fake_tokenizer()
params = SimpleNamespace(extra_args=None)
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params)
assert out.extra_args is None
assert params.extra_args is None
def test_scale_one_leaves_params_byte_identical(self):
serving = self._serving_with_fake_tokenizer()
params = SimpleNamespace(extra_args={"cfg_scale": 1.0})
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params)
assert out.extra_args == {}
def test_guided_request_gains_pair_contract(self):
serving = self._serving_with_fake_tokenizer()
params = SimpleNamespace(extra_args={"cfg_scale": 1.5})
cond = self._cond_prompt("Some transcription text here.")
out = serving._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params)
extra = out.extra_args
assert extra["cfg_scale"] == 1.5
assert extra["cfg_role"] == "cond"
assert extra["cfg_pair_id"] == "req-42"
null_prompt = extra["cfg_null_prompt"]
assert "<unk>" in null_prompt
assert "Some transcription" not in null_prompt
tok = _MarkerTokenizer()
assert len(tok.encode(null_prompt)) == len(tok.encode(cond))
def test_missing_text_prompt_raises(self):
serving = self._serving_with_fake_tokenizer()
params = SimpleNamespace(extra_args={"cfg_scale": 1.5})
with pytest.raises(ValueError, match="adapter-built text prompt"):
serving._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params)
def test_injection_never_mutates_input_params(self):
"""The input may be the engine's SHARED default params (review P2):
pair state must land only on the returned clone."""
serving = self._serving_with_fake_tokenizer()
shared = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
out = serving._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared)
assert out is not shared
assert shared.extra_args == {"cfg_scale": 1.5}
assert shared.temperature == 0.1
assert out.extra_args["cfg_pair_id"] == "req-7"
assert out.temperature == 0.05
class TestAudexStageDetection:
"""audex_omni is speech-capable only alongside the speech decoder."""
def _serving_with_stages(self, stages: list[str]) -> OmniOpenAIServingSpeech:
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
serving.engine_client = SimpleNamespace(
stage_configs=[SimpleNamespace(engine_args=SimpleNamespace(model_stage=s)) for s in stages]
)
return serving
def test_full_pipeline_detected_as_tts(self):
serving = self._serving_with_stages(["audex_omni", "audex_code2wav"])
stage = serving._find_tts_stage()
assert stage is not None and stage.engine_args.model_stage == "audex_omni"
def test_thinker_only_not_detected_as_tts(self):
serving = self._serving_with_stages(["audex_omni"])
assert serving._find_tts_stage() is None
def test_tts_pipeline_still_detected(self):
serving = self._serving_with_stages(["audex_thinker", "audex_code2wav"])
stage = serving._find_tts_stage()
assert stage is not None and stage.engine_args.model_stage == "audex_thinker"
def test_guided_injection_sets_guided_temperature():
"""Guided requests decode at the measured CFG temperature (0.05)."""
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
cond = TestAudexCfgInjection()._cond_prompt()
params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params)
assert out.temperature == 0.05
unguided = SimpleNamespace(extra_args={"cfg_scale": 1.0}, temperature=0.1)
out = serving._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided)
assert out.temperature == 0.1
# ---------------------------------------------------------------- TTA serving surface
from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter # noqa: E402
from vllm_omni.entrypoints.openai.tts_adapters.audex_tta import AudexTTAAdapter # noqa: E402
def _tta_adapter() -> AudexTTAAdapter:
return AudexTTAAdapter.__new__(AudexTTAAdapter)
class _TTAVocabTokenizer(_MarkerTokenizer):
"""Marker tokenizer + a full audiocodec vocab for phase-id building."""
def get_vocab(self):
vocab = {"<audiogen_start>": 7, "<audiogen_end>": 8}
for n in range(8192):
vocab[f"<audiocodec_{n}>"] = 1000 + n
return vocab
class TestAudexTTAAdapterValidation:
def test_default_caption_accepted(self):
assert _tta_adapter().validate(_speech_request()) is None
def test_empty_caption_rejected(self):
req = _speech_request()
req.input = " "
assert _tta_adapter().validate(req) is not None
@pytest.mark.parametrize("scale", [1.0, 3.0, 10.0])
def test_in_range_scale_accepted(self, scale):
assert _tta_adapter().validate(_speech_request(cfg_scale=scale)) is None
@pytest.mark.parametrize("scale", [0.5, 10.5, "big"])
def test_out_of_range_scale_rejected(self, scale):
error = _tta_adapter().validate(_speech_request(cfg_scale=scale))
assert error is not None and "cfg_scale" in error
@pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt", "tta_rvq"])
def test_internal_keys_rejected(self, key):
error = _tta_adapter().validate(_speech_request(**{key: "x"}))
assert error is not None and key in error
class TestAdapterRouting:
def test_tta_adapter_resolves_by_model_type(self):
adapter_cls = resolve_adapter("audex_tta")
assert adapter_cls is AudexTTAAdapter
assert adapter_cls.stage_keys == frozenset({"audex_tta_thinker"})
def test_tts_adapter_never_selects_tta_stage(self):
assert "audex_tta_thinker" not in AudexAdapter.stage_keys
assert not (AudexAdapter.stage_keys & AudexTTAAdapter.stage_keys)
def test_detection_separates_tts_and_tta(self):
tta_serving = TestAudexStageDetection()._serving_with_stages(["audex_tta_thinker", "audex_xcodec"])
tta_serving._tts_stage = tta_serving._find_tts_stage()
assert tta_serving._tts_stage.engine_args.model_stage == "audex_tta_thinker"
assert tta_serving._detect_tts_model_type() == "audex_tta"
tts_serving = TestAudexStageDetection()._serving_with_stages(["audex_thinker", "audex_code2wav"])
tts_serving._tts_stage = tts_serving._find_tts_stage()
assert tts_serving._detect_tts_model_type() == "audex"
class TestAudexTTAInjection:
def _serving(self):
serving = _serving(model_type="audex_tta")
serving._get_audex_tokenizer = lambda: _TTAVocabTokenizer()
serving._audex_tta_rvq = None
return serving
def _prompt(self):
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt
return {"prompt": build_tta_cond_prompt("Rain on a tin roof.")}
def test_default_injection_applies_rvq_and_cfg3(self):
serving = self._serving()
params = SimpleNamespace(extra_args=None)
out = serving._inject_audex_tta_args("req-9", self._prompt(), params)
extra = out.extra_args
rvq = extra["tta_rvq"]
assert len(rvq["phase_token_ids"]) == 4
assert all(len(ids) == 1024 for ids in rvq["phase_token_ids"])
assert (rvq["start_tid"], rvq["end_tid"]) == (7, 8)
assert rvq["codec_cap"] == 4000 and rvq["start_in_prompt"] is True
assert extra["cfg_scale"] == 3.0
assert extra["cfg_role"] == "cond" and extra["cfg_pair_id"] == "req-9"
assert "<unk>" in extra["cfg_null_prompt"]
def test_explicit_scale_one_disables_cfg_but_keeps_rvq(self):
serving = self._serving()
params = SimpleNamespace(extra_args={"cfg_scale": 1.0})
out = serving._inject_audex_tta_args("req-1", self._prompt(), params)
assert "tta_rvq" in out.extra_args
assert "cfg_scale" not in out.extra_args
assert "cfg_role" not in out.extra_args
def test_rvq_contract_cached_per_process(self):
serving = self._serving()
out_a = serving._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None))
out_b = serving._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None))
assert out_a.extra_args["tta_rvq"] is out_b.extra_args["tta_rvq"]
def test_injection_never_mutates_input_params(self):
"""The default online path hands the injector an element of the
engine's SHARED default_sampling_params_list (review P2): the
per-request pair state must land only on the returned clone."""
serving = self._serving()
shared = SimpleNamespace(extra_args=None)
out_1 = serving._inject_audex_tta_args("req-1", self._prompt(), shared)
out_2 = serving._inject_audex_tta_args("req-2", self._prompt(), shared)
assert shared.extra_args is None
assert out_1 is not shared and out_2 is not shared
assert out_1.extra_args["cfg_pair_id"] == "req-1"
assert out_2.extra_args["cfg_pair_id"] == "req-2"
+21
View File
@@ -529,6 +529,26 @@ def decode_b64_image(b64: str):
return img
def concat_audio(audio_val) -> np.ndarray:
"""Flatten a multimodal audio payload to mono float32 samples.
Engines return ``multimodal_output["audio"]`` as a tensor, a list of
per-chunk tensors (streaming decoders), or an array-like; concatenate
in order and return a 1-D ``np.float32`` array (empty when a list has
no tensors).
"""
import torch
if isinstance(audio_val, list):
tensors = [t.detach().cpu().float().reshape(-1) for t in audio_val if isinstance(t, torch.Tensor)]
if not tensors:
return np.zeros((0,), dtype=np.float32)
return torch.cat(tensors, dim=-1).numpy().astype(np.float32, copy=False)
if isinstance(audio_val, torch.Tensor):
return audio_val.detach().cpu().float().reshape(-1).numpy()
return np.asarray(audio_val, dtype=np.float32).reshape(-1)
def preprocess_text(text):
import opencc
@@ -695,6 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") ->
__all__ = [
"_merge_base64_audio_to_segment",
"concat_audio",
"convert_audio_bytes_to_text",
"convert_audio_file_to_text",
"cosine_similarity_text",