[New Model] support nemotron Audex (#4976)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
64 files changed
+7760
-39
No files matched your search
@@ -392,6 +392,11 @@ def test_sub_config_fields_match_rfc_scopes():
|
||||
"enable_multithread_weight_load",
|
||||
"num_weight_load_threads",
|
||||
"disable_autocast",
|
||||
# Per-stage checkpoint resolution for repos whose stages live in
|
||||
# subfolders (e.g. Audex): mirrors StagePipelineConfig on the
|
||||
# legacy engine-args path.
|
||||
"model_subdir",
|
||||
"tokenizer_subdir",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageLoadConfig)} == {
|
||||
"load_format",
|
||||
|
||||
@@ -121,6 +121,10 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
|
||||
("Qwen3TTSCode2Wav", "code2wav"),
|
||||
("MiniCPMO45Code2Wav", "code2wav"),
|
||||
("CosyVoice3Model", "cosyvoice3_code2wav"),
|
||||
# audex: sync-path TTS decoder (async_chunk: false deploys) and
|
||||
# the TTA XCodec1 stage (always sync full-payload).
|
||||
("AudexCode2Wav", "audex_code2wav"),
|
||||
("AudexXCodec1", "audex_xcodec"),
|
||||
("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"),
|
||||
("DyninOmniForConditionalGeneration", "token2image"),
|
||||
("DyninOmniForConditionalGeneration", "token2audio"),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HTTP + WebSocket validation for Qwen3-TTS Base and Higgs Audio V3 ``/v1/audio/speech`` (and related routes)."""
|
||||
"""HTTP + WebSocket validation for Qwen3-TTS Base, Higgs Audio V3 and Audex TTA ``/v1/audio/speech`` (and related routes)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -70,6 +70,19 @@ _HIGGS_AUDIO_V3_SPEECH = [
|
||||
),
|
||||
]
|
||||
|
||||
_AUDEX_TTA_CAPTION = "Heavy rain falling on a tin roof."
|
||||
_AUDEX_TTA_SPEECH = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="nvidia/Nemotron-Labs-Audex-2B",
|
||||
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
|
||||
server_args=_SPEECH_SERVER_ARGS,
|
||||
),
|
||||
id="audex_tta",
|
||||
marks=_L4_SPEECH_HW,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
|
||||
if loc == "batch":
|
||||
@@ -325,6 +338,37 @@ def test_speech_higgs_invalid_field_values(
|
||||
)
|
||||
|
||||
|
||||
# Audex TTA (``audex_tta`` deployment): the serving layer builds the caption
|
||||
# prompt and injects the RVQ phase contract + default guidance server-side, so
|
||||
# guidance knobs and speech-only fields must be rejected before generation.
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, err_message",
|
||||
[
|
||||
pytest.param({"extra_params": {"cfg_scale": 99}}, "cfg_scale", id="cfg_scale_out_of_range"),
|
||||
pytest.param({"extra_params": {"cfg_scale": "big"}}, "cfg_scale", id="cfg_scale_wrong_type"),
|
||||
pytest.param({"extra_params": {"tta_rvq": {}}}, "tta_rvq", id="tta_rvq_not_overridable"),
|
||||
pytest.param({"input": " "}, "caption", id="caption_whitespace_only"),
|
||||
pytest.param({"voice": "alloy"}, "voice", id="voice_not_supported"),
|
||||
pytest.param(
|
||||
{"ref_audio": "https://example.com/ref.wav"},
|
||||
"reference audio",
|
||||
id="ref_audio_not_supported",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _AUDEX_TTA_SPEECH, indirect=True)
|
||||
def test_speech_audex_tta_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
body = {"model": omni_server.model, "input": _AUDEX_TTA_CAPTION, **overrides}
|
||||
openai_client.send_audio_speech_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/audio/speech/batch (JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E test for the Audex cascaded speech-to-speech pipeline.
|
||||
|
||||
One ``audex_s2s`` deployment serves all three official
|
||||
passes. The routing contract is the hard gate here: text-modality passes
|
||||
(ASR, chat) finish at stage 0 and must never produce stage-1 audio; the
|
||||
audio-modality TTS pass streams through the causal speech decoder.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
ASR_PROMPT = (
|
||||
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
)
|
||||
|
||||
# Official chat-pass recipe (examples/offline_inference/audex/speech_to_speech.py,
|
||||
# mirroring the official cascaded web demo) — 30B-A3B ONLY. The
|
||||
# formatting-instruction prefix is load-bearing on the 30B-A3B: without it a bare
|
||||
# transcript turn is treated as a TTS request and the model emits literal
|
||||
# <speechcodec_N> tokens instead of text, and under the yaml's near-greedy
|
||||
# defaults its reasoning block never closes. The 2B must NOT use this recipe:
|
||||
# it is long-verified stable on a bare prompt + engine-default sampling, while
|
||||
# the official temp-1.0 seed-0 draw hallucinates off-language text (yet closes
|
||||
# </think> cleanly, so the seed retry never fires) and fails the round-trip
|
||||
# WER gate — measured, reproduced byte-identically.
|
||||
SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
|
||||
TEXT_INSTRUCTION = (
|
||||
"SYSTEM & FORMATTING INSTRUCTION: You are Audex, created by NVIDIA based on the Nemotron-Cascade-2 architecture. "
|
||||
"You may output your reasoning, followed by your final response. "
|
||||
"You may format your reasoning block however you like. "
|
||||
"However, your final response must strictly follow these rules: "
|
||||
"* Write in plain, unformatted prose like a book or newspaper article. "
|
||||
"* Do not use markdown, bullet points, lists, or headers. "
|
||||
"* Standard numbers, abbreviations, and symbols are acceptable. "
|
||||
"* [CRITICAL] You must press enter after every single sentence, placing each sentence on its own separate line."
|
||||
)
|
||||
CHAT_MAX_TOKENS = 2048 # official --text-max-tokens default
|
||||
# The official demo samples this pass unseeded; some trajectories deliberate
|
||||
# past the token budget without ever closing the think block, so retry a few
|
||||
# deterministic seeds until one closes.
|
||||
CHAT_SEEDS = (0, 1, 2)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_s2s.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": True}),
|
||||
id="s2s_full",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_s2s_30b.yaml"), {"async_chunk": True}),
|
||||
id="s2s_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(text: str) -> list[str]:
|
||||
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
|
||||
|
||||
|
||||
def _wer(hyp: str, ref: str) -> float:
|
||||
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
|
||||
if not ref_words:
|
||||
return 0.0
|
||||
rows = list(range(len(hyp_words) + 1))
|
||||
for i, ref_word in enumerate(ref_words, 1):
|
||||
prev, rows[0] = rows[0], i
|
||||
for j, hyp_word in enumerate(hyp_words, 1):
|
||||
cur = min(rows[j] + 1, rows[j - 1] + 1, prev + (ref_word != hyp_word))
|
||||
prev, rows[j] = rows[j], cur
|
||||
return rows[-1] / len(ref_words)
|
||||
|
||||
|
||||
def _chat_prompt(user_text: str) -> str:
|
||||
# Official chat-pass recipe: system prompt + formatting instruction
|
||||
# prefixed to the user text, thinking ENABLED via an open <think> block
|
||||
# (chat_template.jinja with enable_thinking=True renders
|
||||
# "<|im_start|>assistant\n<think>\n"). Priming the block open keeps the
|
||||
# reasoning inside <think>...</think> so _extract_final_response can drop
|
||||
# it; with a closed block the 30B leaks the reasoning as plain text.
|
||||
return (
|
||||
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{TEXT_INSTRUCTION}\n\n{user_text}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n<think>\n"
|
||||
)
|
||||
|
||||
|
||||
def _extract_final_response(text: str) -> str:
|
||||
# Mirror the official demo's extract_spoken_answer: keep only what
|
||||
# follows the LAST closing think tag, falling back to the full text when
|
||||
# the model skipped reasoning or never closed the block. Benign no-op for
|
||||
# the 2B, whose answers carry no think block.
|
||||
if "</think>" in text:
|
||||
answer = text.rsplit("</think>", 1)[-1].strip()
|
||||
if answer:
|
||||
return answer
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _chat_params(runner: OmniRunner, seed: int):
|
||||
# Official demo text-pass sampling (--text-temperature 1.0, --text-top-p
|
||||
# 0.95, --text-max-tokens 2048). The yaml stage-0 defaults (temperature
|
||||
# 0.1, top_k 80) are TTS-thinker settings; under near-greedy decoding the
|
||||
# 30B loops inside its reasoning block until the token budget runs out
|
||||
# and never emits </think>.
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
stage0.temperature = 1.0
|
||||
stage0.top_p = 0.95
|
||||
stage0.top_k = -1
|
||||
stage0.max_tokens = CHAT_MAX_TOKENS
|
||||
stage0.seed = seed
|
||||
return params
|
||||
|
||||
|
||||
def _chat_once(runner: OmniRunner, prompt: str, seed: int) -> tuple[str, bool]:
|
||||
(req_output,) = runner.omni.generate([{"prompt": prompt, "modalities": ["text"]}], _chat_params(runner, seed))
|
||||
out = req_output.outputs[0]
|
||||
text = (out.text or "").strip()
|
||||
truncated = getattr(out, "finish_reason", None) == "length" or len(out.token_ids or []) >= CHAT_MAX_TOKENS
|
||||
return text, truncated
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_audex_s2s_cascade_round_trip(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""ASR → chat → TTS over one deployment; text passes never emit audio."""
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
# Spoken question: vLLM's public audio asset (downloaded on first use).
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
# Pass 1 — ASR (text-final).
|
||||
asr_outputs = omni_runner.omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": ASR_PROMPT,
|
||||
"multi_modal_data": {"audio": (audio, sr)},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
]
|
||||
)
|
||||
assert len(asr_outputs) == 1
|
||||
asr_out = asr_outputs[0]
|
||||
transcript = (asr_out.outputs[0].text or "").strip()
|
||||
mm = getattr(asr_out, "multimodal_output", None) or {}
|
||||
stage1_audio = concat_audio(mm.get("audio")) if "audio" in mm else np.zeros((0,), dtype=np.float32)
|
||||
assert stage1_audio.size == 0, "Text-modality ASR pass produced stage-1 audio (routing regression)"
|
||||
|
||||
# Pass 2 — chat (text-final). The recipe is size-conditional (see the
|
||||
# SYSTEM_PROMPT comment above): the 30B requires the official recipe,
|
||||
# the 2B regresses under it.
|
||||
if omni_runner.model_name == _AUDEX_30B_MODEL:
|
||||
# Official recipe. The raw completion is
|
||||
# <reasoning...></think><answer>; speak only the final response.
|
||||
# Retry with the next seed only when the budget ran out before the
|
||||
# reasoning block closed (a natural stop without </think> is a
|
||||
# complete answer); if every seed spins, keep the full text
|
||||
# (official demo behavior).
|
||||
chat_prompt = _chat_prompt(transcript or "Hello.")
|
||||
raw = ""
|
||||
for seed in CHAT_SEEDS:
|
||||
raw, truncated = _chat_once(omni_runner, chat_prompt, seed)
|
||||
if "</think>" in raw or not truncated:
|
||||
break
|
||||
answer = _extract_final_response(raw)
|
||||
else:
|
||||
# 2B: bare prompt with a closed think block, engine-default sampling
|
||||
# (yaml near-greedy), no retry; the answer is used as-is.
|
||||
chat_prompt = f"<|im_start|>user\n{transcript or 'Hello.'}<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
(chat_output,) = omni_runner.omni.generate([{"prompt": chat_prompt, "modalities": ["text"]}])
|
||||
answer = (chat_output.outputs[0].text or "").strip()
|
||||
|
||||
if real_weights:
|
||||
assert transcript, "ASR pass produced an empty transcript"
|
||||
assert answer, "Chat pass produced an empty answer"
|
||||
|
||||
# Pass 3 — TTS (audio-final; streams through code2wav). Both final
|
||||
# stages may emit an output record; collect the audio-bearing one.
|
||||
# The formatting instruction puts every sentence on its own line;
|
||||
# flatten to plain prose for the TTS thinker.
|
||||
tts_text = re.sub(r"\s+", " ", answer).strip() if answer else "The weather is nice today."
|
||||
tts_text = tts_text[:200]
|
||||
tts_prompt = {"prompt": build_cond_prompt(tts_text), "modalities": ["audio"]}
|
||||
tts_outputs = omni_runner.omni.generate([tts_prompt])
|
||||
assert tts_outputs, "TTS pass returned no outputs"
|
||||
speech = np.zeros((0,), dtype=np.float32)
|
||||
for req_output in tts_outputs:
|
||||
mm = getattr(req_output, "multimodal_output", None) or {}
|
||||
if "audio" in mm:
|
||||
speech = concat_audio(mm["audio"])
|
||||
break
|
||||
assert speech.size > 0, "TTS pass produced empty audio"
|
||||
|
||||
if real_weights:
|
||||
rms = float(np.sqrt(np.mean(np.square(speech))))
|
||||
assert rms > 1e-3, f"TTS-pass audio near-silent (rms={rms})"
|
||||
|
||||
# Pass 4 — output verification: transcribe the synthesized answer
|
||||
# with the SAME deployment's ASR pass and compare to the answer
|
||||
# text (self-contained; no external ASR dependency).
|
||||
verify_outputs = omni_runner.omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": ASR_PROMPT,
|
||||
"multi_modal_data": {"audio": (speech, SAMPLE_RATE)},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
]
|
||||
)
|
||||
verify_text = (verify_outputs[0].outputs[0].text or "").strip()
|
||||
assert verify_text, "Output-verification ASR pass returned empty text"
|
||||
# The full checkpoint prefixes transcripts with a language-tag
|
||||
# sentence and quotes the content; extract the quoted payload.
|
||||
if "'" in verify_text and len(verify_text.split("'")) >= 3:
|
||||
verify_text = verify_text.split("'")[1].strip()
|
||||
wer = _wer(verify_text, tts_text)
|
||||
# Hard gate: the synthesized WAV must transcribe back to the answer
|
||||
# (lenient threshold absorbs ASR quoting/prefix differences).
|
||||
assert wer <= 0.5, f"Output WAV does not match the answer (WER={wer:.2f}): {verify_text!r} vs {tts_text!r}"
|
||||
print(f"[trend] Audex S2S output-ASR WER vs answer: {wer:.3f} ({verify_text!r})")
|
||||
|
||||
# Final pass — time-to-first-audio for the streaming TTS path. This
|
||||
# MUST be the last engine call: the py_generator API closes the engine
|
||||
# when the generator is exhausted.
|
||||
t_start = time.perf_counter()
|
||||
first_audio_s: float | None = None
|
||||
for req_output in omni_runner.omni.generate([tts_prompt], py_generator=True):
|
||||
mm = getattr(req_output, "multimodal_output", None) or {}
|
||||
if "audio" in mm and concat_audio(mm["audio"]).size > 0:
|
||||
if first_audio_s is None:
|
||||
first_audio_s = time.perf_counter() - t_start
|
||||
assert first_audio_s is not None, "streaming TTS never produced an audio-bearing chunk"
|
||||
print(f"[trend] Audex S2S TTS-pass first-audio latency: {first_audio_s * 1000.0:.0f} ms")
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex audio understanding (thinker-only).
|
||||
|
||||
Single-stage ``audex_thinker_only`` pipeline on
|
||||
``checkpoint_folder_full``: WAV (+ instruction) in, text out. The input clip
|
||||
is vLLM's public ``mary_had_lamb`` audio asset (downloaded on first use), so
|
||||
its transcript is known.
|
||||
|
||||
WER is a trend metric (recorded in the assertion message, no numeric hard
|
||||
gate); the hard gates are structural: non-empty, non-degenerate text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
|
||||
# Full transcript of the mary_had_lamb asset (Edison's phonograph recording);
|
||||
# a shortened reference would count the extra spoken words as insertions and
|
||||
# inflate WER past the trend threshold.
|
||||
_REFERENCE = (
|
||||
"the first words i spoke in the original phonograph a little piece of practical poetry "
|
||||
"mary had a little lamb its fleece was white as snow and everywhere that mary went the "
|
||||
"lamb was sure to go"
|
||||
)
|
||||
|
||||
ASR_PROMPT = (
|
||||
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_thinker_only.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": False}),
|
||||
id="thinker_only",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_thinker_only_30b.yaml"), {"async_chunk": False}),
|
||||
id="thinker_only_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(text: str) -> list[str]:
|
||||
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
|
||||
|
||||
|
||||
def _wer(hyp: str, ref: str) -> float:
|
||||
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
|
||||
if not ref_words:
|
||||
return 0.0
|
||||
rows = list(range(len(hyp_words) + 1))
|
||||
for i, ref_word in enumerate(ref_words, 1):
|
||||
prev, rows[0] = rows[0], i
|
||||
for j, hyp_word in enumerate(hyp_words, 1):
|
||||
cur = min(
|
||||
rows[j] + 1,
|
||||
rows[j - 1] + 1,
|
||||
prev + (ref_word != hyp_word),
|
||||
)
|
||||
prev, rows[j] = rows[j], cur
|
||||
return rows[-1] / len(ref_words)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_asr_smoke(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""Transcribing the public audio asset yields coherent, non-degenerate text."""
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
outputs = omni_runner.omni.generate([{"prompt": ASR_PROMPT, "multi_modal_data": {"audio": (audio, sr)}}])
|
||||
|
||||
assert len(outputs) == 1
|
||||
text = outputs[0].outputs[0].text if outputs[0].outputs else ""
|
||||
assert isinstance(text, str)
|
||||
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
words = _normalize(text)
|
||||
assert words, f"Empty transcription: {text!r}"
|
||||
# Degenerate repetition guard: no word may dominate the output.
|
||||
most_common = max(words.count(w) for w in set(words))
|
||||
assert most_common <= max(3, len(words) // 2), f"Degenerate transcript: {text!r}"
|
||||
wer = _wer(text, _REFERENCE)
|
||||
# Trend metric (recorded, not gated): substantially-correct check only.
|
||||
assert wer <= 0.9, f"Transcript unrelated to reference (WER={wer:.2f}): {text!r}"
|
||||
print(f"[trend] Audex S2T WER vs reference: {wer:.3f} ({text!r})")
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex text-to-audio (TTA).
|
||||
|
||||
Caption → RVQ-phase-masked <audiocodec_*> tokens → XCodec1 waveform through
|
||||
the ``audex_tta`` pipeline. RVQ phase validity is enforced by
|
||||
the mask logits processor (unit-tested against the official validator); the
|
||||
hard gates here are decode-side: non-silent, finite audio at the documented
|
||||
sample rate. CFG runs at the official TTA default (3.0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
# The acceptance gate requires eight captions decoding to non-silent,
|
||||
# finite audio through XCodec1.
|
||||
CAPTIONS = (
|
||||
"Heavy rain falling on a tin roof.",
|
||||
"A dog barking in the distance while birds chirp.",
|
||||
"Ocean waves crashing on a rocky shore.",
|
||||
"A crackling campfire at night with crickets.",
|
||||
"Thunder rumbling during a storm.",
|
||||
"A steam train whistle blowing as it departs.",
|
||||
"Wind chimes ringing in a gentle breeze.",
|
||||
"Footsteps crunching through fresh snow.",
|
||||
)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_tta.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": False}),
|
||||
id="tta_full_payload",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tta_30b.yaml"), {"async_chunk": False}),
|
||||
id="tta_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tta_tokenizer_bits():
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids
|
||||
|
||||
root = ensure_audex_snapshot(_audex_model, profile="tta")
|
||||
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
|
||||
phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer)
|
||||
return tokenizer, phase_token_ids, start_tid, end_tid
|
||||
|
||||
|
||||
def _tta_sampling_params(runner: OmniRunner, cond_prompt: str, tokenizer_bits, cfg_scale: float = 3.0):
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt
|
||||
|
||||
tokenizer, phase_token_ids, start_tid, end_tid = tokenizer_bits
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
if stage0.extra_args is None:
|
||||
stage0.extra_args = {}
|
||||
stage0.extra_args["tta_rvq"] = {
|
||||
"phase_token_ids": phase_token_ids,
|
||||
"start_tid": start_tid,
|
||||
"end_tid": end_tid,
|
||||
"codec_cap": 4000,
|
||||
"start_in_prompt": True,
|
||||
}
|
||||
if cfg_scale > 1.0:
|
||||
stage0.extra_args.update(
|
||||
{
|
||||
"cfg_scale": float(cfg_scale),
|
||||
"cfg_role": "cond",
|
||||
"cfg_pair_id": f"e2e-tta-{abs(hash(cond_prompt)) % 10_000_000}",
|
||||
"cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer),
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_tta_eight_captions(omni_runner: OmniRunner, run_level: str, tta_tokenizer_bits) -> None:
|
||||
"""All eight captions decode through XCodec1 to non-silent, finite audio.
|
||||
|
||||
RVQ phase validity is enforced by the mask logits processor by
|
||||
construction; the final user output carries only decoded audio (the
|
||||
audio-final pipeline emits no stage-0 token record), and disabling the
|
||||
mask in-engine as an e2e control would require a second engine boot
|
||||
with a different logits-processor config. The mask's sensitivity is
|
||||
therefore proven at unit level (see test_audex_tta.py's mask tests and
|
||||
the unmasked-sampling control) against the official validator.
|
||||
"""
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
for caption in CAPTIONS:
|
||||
prompt = build_tta_cond_prompt(caption)
|
||||
params = _tta_sampling_params(omni_runner, prompt, tta_tokenizer_bits)
|
||||
outputs = omni_runner.omni.generate([prompt], params)
|
||||
|
||||
assert len(outputs) == 1, f"{caption!r}: expected one output, got {len(outputs)}"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, f"{caption!r}: no audio output: {list(audio_mm.keys())}"
|
||||
audio = concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, f"{caption!r}: generated audio is empty"
|
||||
assert np.isfinite(audio).all(), f"{caption!r}: audio contains NaN/Inf"
|
||||
|
||||
sr_val = audio_mm.get("sr", SAMPLE_RATE)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
assert int(sr_val) == SAMPLE_RATE, f"{caption!r}: unexpected sample_rate={sr_val}"
|
||||
|
||||
if real_weights:
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
# The decode cap is 500 frames = 10 s at 50 fps.
|
||||
assert 0.1 <= duration_s <= 10.5, f"{caption!r}: unexpected duration={duration_s:.3f}s"
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-4, f"{caption!r}: audio is near-silent (rms={rms})"
|
||||
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex (Nemotron-Labs-Audex-2B) TTS.
|
||||
|
||||
Passes the HF repo ROOT and verifies the 2-stage pipeline (thinker →
|
||||
streaming causal speech decoder): root-path/subfolder resolution, codec-token
|
||||
extraction, per-request streaming sessions, and the lookahead flush at EOS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
SYNTH_TEXTS = (
|
||||
"The weather is so good, and I want to enjoy the beautiful morning in the park.",
|
||||
"The quick brown fox jumps over the lazy dog.",
|
||||
)
|
||||
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_tts.yaml")
|
||||
# Collection must not download anything: pass the repo id (or a local
|
||||
# override) and let the engine's stage-init path resolve/download the
|
||||
# required snapshot subset at execution time (ensure_audex_snapshot).
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": True}),
|
||||
id="async_chunk",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tts_30b.yaml"), {"async_chunk": True}),
|
||||
id="tts_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_tts_smoke(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""Audex TTS from the repo root should produce sane 16 kHz audio per prompt.
|
||||
|
||||
At ``core_model`` level the runner loads DUMMY weights (structural smoke
|
||||
only); real-speech assertions apply from ``advanced_model`` up.
|
||||
"""
|
||||
prompts = [build_cond_prompt(text) for text in SYNTH_TEXTS]
|
||||
outputs = omni_runner.omni.generate(prompts)
|
||||
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
assert len(outputs) == len(SYNTH_TEXTS), f"expected {len(SYNTH_TEXTS)} outputs, got {len(outputs)}"
|
||||
for output in outputs:
|
||||
audio_mm = output.multimodal_output
|
||||
assert "audio" in audio_mm, f"No audio output found: {list(audio_mm.keys())}"
|
||||
audio = concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
|
||||
sr_val = audio_mm.get("sr", SAMPLE_RATE)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
assert int(sr_val) == SAMPLE_RATE, f"Unexpected sample_rate={sr_val}"
|
||||
|
||||
if real_weights:
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
|
||||
# Real speech, not silence.
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-3, f"Audio is near-silent (rms={rms})"
|
||||
|
||||
|
||||
def _cfg_sampling_params(runner: OmniRunner, cfg_scale: float, pair_id: str, cond_prompt: str):
|
||||
"""Stage sampling params carrying the CFG pair contract for one request."""
|
||||
import copy
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_null_prompt
|
||||
|
||||
# Resolve the ACTIVE param's model (2B or 30B), not the module default:
|
||||
# the null prompt must be built with the same tokenizer the engine uses.
|
||||
root = ensure_audex_snapshot(runner.model_name)
|
||||
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
if stage0.extra_args is None:
|
||||
stage0.extra_args = {}
|
||||
stage0.extra_args.update(
|
||||
{
|
||||
"cfg_scale": float(cfg_scale),
|
||||
"cfg_role": "cond",
|
||||
"cfg_pair_id": pair_id,
|
||||
"cfg_null_prompt": build_null_prompt(cond_prompt, tokenizer),
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_cfg_guided_single_stream(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""A guided request must yield exactly ONE audio stream (companion suppressed)."""
|
||||
prompt = build_cond_prompt(SYNTH_TEXTS[0])
|
||||
params = _cfg_sampling_params(omni_runner, 1.5, "e2e-cfg-pair", prompt)
|
||||
outputs = omni_runner.omni.generate([prompt], params)
|
||||
|
||||
assert len(outputs) == 1, f"expected exactly one output (no companion leak), got {len(outputs)}"
|
||||
audio = concat_audio(outputs[0].multimodal_output["audio"])
|
||||
assert audio.size > 0, "Guided generation produced empty audio"
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-3, f"Guided audio is near-silent (rms={rms})"
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_cfg_disabled_is_byte_identical(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""cfg_scale=1.0 must be indistinguishable from not passing cfg at all."""
|
||||
prompt = build_cond_prompt(SYNTH_TEXTS[1])
|
||||
|
||||
baseline = omni_runner.omni.generate([prompt])
|
||||
base_audio = concat_audio(baseline[0].multimodal_output["audio"])
|
||||
|
||||
import copy
|
||||
|
||||
params = copy.deepcopy(omni_runner.omni.resolve_sampling_params_list(None))
|
||||
if params[0].extra_args is None:
|
||||
params[0].extra_args = {}
|
||||
params[0].extra_args["cfg_scale"] = 1.0
|
||||
disabled = omni_runner.omni.generate([prompt], params)
|
||||
disabled_audio = concat_audio(disabled[0].multimodal_output["audio"])
|
||||
|
||||
assert base_audio.shape == disabled_audio.shape, (
|
||||
f"cfg_scale=1.0 changed output length: {base_audio.shape} vs {disabled_audio.shape}"
|
||||
)
|
||||
np.testing.assert_array_equal(base_audio, disabled_audio)
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) text-to-audio.
|
||||
|
||||
Covers /v1/audio/speech through the ``audex_tta`` deployment:
|
||||
the ``audex_tta`` adapter builds the caption prompt and the serving layer
|
||||
injects the RVQ phase contract plus the official default guidance
|
||||
(cfg_scale 3.0) server-side.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
CAPTION = "Heavy rain falling on a tin roof."
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
audex_tta_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="audex_tta",
|
||||
)
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
audex_tta_server_params.append(
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_AUDEX_30B_MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tta_30b.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="online_tta_30b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_tta_server_params, indirect=True)
|
||||
def test_audex_tta_online_smoke(omni_server, openai_client, run_level) -> None:
|
||||
"""A caption returns a valid finite non-silent WAV at the default cfg 3.0."""
|
||||
import io
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={"model": omni_server.model, "input": CAPTION, "response_format": "wav"},
|
||||
timeout=600,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text[:200]
|
||||
assert resp.content[:4] == b"RIFF", "not a WAV response"
|
||||
|
||||
with wave.open(io.BytesIO(resp.content)) as wav:
|
||||
assert wav.getframerate() == SAMPLE_RATE, f"unexpected sample rate {wav.getframerate()}"
|
||||
frames = wav.readframes(wav.getnframes())
|
||||
duration_s = wav.getnframes() / wav.getframerate()
|
||||
|
||||
audio = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
assert audio.size > 0, "empty audio body"
|
||||
assert np.isfinite(audio).all(), "audio contains NaN/Inf"
|
||||
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
# The decode cap is 500 frames = 10 s at 50 fps.
|
||||
assert 0.1 <= duration_s <= 10.5, f"unexpected duration={duration_s:.3f}s"
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-4, f"audio is near-silent (rms={rms})"
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) TTS.
|
||||
|
||||
Covers /v1/audio/speech non-streaming and audio-byte streaming, plus the
|
||||
Audex request policies: single built-in voice (missing/empty/"default" only),
|
||||
no reference audio, and cfg_scale reserved at 1.0.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
PROMPT = "The weather is so good, and I want to enjoy the beautiful morning in the park."
|
||||
|
||||
audex_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tts.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="audex",
|
||||
)
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
audex_server_params.append(
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_AUDEX_30B_MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tts_30b.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="online_tts_30b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_nonstream(omni_server, openai_client) -> None:
|
||||
"""Plain English TTS, non-streaming WAV response."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_stream_audio(omni_server, openai_client) -> None:
|
||||
"""Plain English TTS, raw audio-byte streaming."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_max_new_tokens_caps_duration(omni_server, openai_client, run_level) -> None:
|
||||
"""max_new_tokens must cap stage-0 generation (review P2 regression)."""
|
||||
import io
|
||||
import wave
|
||||
|
||||
import requests
|
||||
|
||||
if run_level not in {"advanced_model", "full_model"}:
|
||||
pytest.skip("duration cap needs real weights")
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"response_format": "wav",
|
||||
"max_new_tokens": 25,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
with wave.open(io.BytesIO(resp.content)) as wav:
|
||||
duration_s = wav.getnframes() / wav.getframerate()
|
||||
# 25 codec frames at 50 fps = 0.5 s (+ small lookahead tail); the full
|
||||
# sentence would be ~3 s, so a capped result proves the override applied.
|
||||
assert duration_s <= 1.0, f"max_new_tokens ignored: duration={duration_s:.2f}s"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_request_policies(omni_server, openai_client) -> None:
|
||||
"""Unsupported voice / CFG / empty input must be rejected with 400."""
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
|
||||
ok = requests.post(url, json={"model": omni_server.model, "input": PROMPT, "voice": "default"})
|
||||
assert ok.status_code == 200, ok.text
|
||||
|
||||
for bad_payload, needle in (
|
||||
({"input": PROMPT, "voice": "alloy"}, "voice"),
|
||||
({"input": " "}, "non-empty"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": 0.5}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": 10.5}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": "big"}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_role": "cond"}}, "cfg_role"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_pair_id": "x"}}, "cfg_pair_id"),
|
||||
({"input": PROMPT, "ref_audio": "https://example.com/ref.wav"}, "reference audio"),
|
||||
):
|
||||
resp = requests.post(url, json={"model": omni_server.model, **bad_payload})
|
||||
assert resp.status_code == 400, f"{bad_payload} -> {resp.status_code}: {resp.text[:200]}"
|
||||
assert needle in resp.text, f"{bad_payload} error missing {needle!r}: {resp.text[:200]}"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_cfg_nonstream(omni_server, openai_client) -> None:
|
||||
"""Guided (cfg_scale=1.5) non-streaming request returns one WAV."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_cfg_stream_audio(omni_server, openai_client) -> None:
|
||||
"""Guided (cfg_scale=1.5) streaming request yields a single audio stream."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_mixed_cfg_concurrency(omni_server, openai_client) -> None:
|
||||
"""Mixed guided/unguided requests at concurrency 4 all complete with audio."""
|
||||
import concurrent.futures
|
||||
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
payloads = [
|
||||
{"model": omni_server.model, "input": f"{PROMPT} Take {i}.", "response_format": "wav"}
|
||||
| ({"extra_params": {"cfg_scale": 1.5}} if i % 2 == 0 else {})
|
||||
for i in range(4)
|
||||
]
|
||||
|
||||
def _post(payload):
|
||||
return requests.post(url, json=payload, timeout=300)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
responses = list(pool.map(_post, payloads))
|
||||
|
||||
for payload, resp in zip(payloads, responses):
|
||||
assert resp.status_code == 200, f"{payload} -> {resp.status_code}: {resp.text[:200]}"
|
||||
assert resp.content[:4] == b"RIFF", f"{payload}: not a WAV response"
|
||||
assert len(resp.content) > 44, f"{payload}: empty audio body"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_stream_abort_leaves_server_healthy(omni_server, openai_client) -> None:
|
||||
"""Aborting a guided streaming request mid-flight must not wedge the server.
|
||||
|
||||
The abort cascades to the CFG companion (cfg_companion_tracker); a
|
||||
follow-up guided request on the same server must still succeed.
|
||||
"""
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
long_input = " ".join([PROMPT] * 4)
|
||||
|
||||
with requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": long_input,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
},
|
||||
stream=True,
|
||||
timeout=300,
|
||||
) as resp:
|
||||
assert resp.status_code == 200, resp.text
|
||||
# Read one chunk, then abort the connection mid-stream.
|
||||
next(resp.iter_content(chunk_size=4096))
|
||||
|
||||
follow_up = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
assert follow_up.status_code == 200, follow_up.text[:200]
|
||||
assert follow_up.content[:4] == b"RIFF" and len(follow_up.content) > 44
|
||||
@@ -0,0 +1,376 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Serving-layer regressions for the Audex zero-codec-token contract.
|
||||
|
||||
A zero-codec-token (or phase-invalid TTA) request reaches the serving layer
|
||||
as a stream whose only audio payloads are zero-size tensors. The streaming
|
||||
generator must not turn that into a header-only/empty successful stream: the
|
||||
raw-bytes generator raises (before any WAV header), and the SSE wrapper
|
||||
surfaces ``speech.audio.error`` instead of ``speech.audio.done``. The
|
||||
contract covers both audex pipelines (``audex`` TTS and ``audex_tta``).
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES,
|
||||
OmniOpenAIServingSpeech,
|
||||
)
|
||||
|
||||
|
||||
def _serving(model_type: str = "audex") -> OmniOpenAIServingSpeech:
|
||||
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
serving._tts_model_type = model_type
|
||||
serving._mark_ref_audio_artifact_ready_for_request = lambda request_id: None
|
||||
serving._discard_ref_audio_artifact_warmup = lambda request_id: None
|
||||
return serving
|
||||
|
||||
|
||||
def _result(audio: torch.Tensor) -> SimpleNamespace:
|
||||
return SimpleNamespace(multimodal_output={"model_outputs": audio, "sr": torch.tensor(16000, dtype=torch.int32)})
|
||||
|
||||
|
||||
async def _gen(*results):
|
||||
for res in results:
|
||||
yield res
|
||||
|
||||
|
||||
async def _collect(chunks):
|
||||
out = []
|
||||
async for chunk in chunks:
|
||||
out.append(chunk)
|
||||
return out
|
||||
|
||||
|
||||
def test_no_audio_guard_covers_both_audex_pipelines():
|
||||
# The non-streaming CreateAudio guard keys off the same constant; pin
|
||||
# its membership so neither pipeline regresses to empty-200 responses.
|
||||
assert _AUDEX_NO_AUDIO_GUARD_MODEL_TYPES == {"audex", "audex_tta"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_type", ["audex", "audex_tta"])
|
||||
async def test_raw_stream_raises_on_empty_audio_before_any_header(model_type):
|
||||
serving = _serving(model_type)
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_gen(_result(torch.empty(0)), _result(torch.empty(0))),
|
||||
request_id="req-empty",
|
||||
response_format="wav",
|
||||
)
|
||||
with pytest.raises(ValueError, match="no audio"):
|
||||
await _collect(chunks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_stream_yields_audio_normally_when_non_empty():
|
||||
serving = _serving()
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_gen(_result(torch.zeros(320)), _result(torch.empty(0))),
|
||||
request_id="req-ok",
|
||||
response_format="wav",
|
||||
)
|
||||
collected = await _collect(chunks)
|
||||
# WAV header + one PCM chunk; the trailing empty chunk is skipped silently.
|
||||
assert len(collected) == 2
|
||||
assert collected[0][:4] == b"RIFF"
|
||||
assert len(collected[1]) == 320 * 2 # int16 PCM
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_type", ["audex", "audex_tta"])
|
||||
async def test_sse_stream_emits_error_not_done_on_empty_audio(model_type):
|
||||
serving = _serving(model_type)
|
||||
events = serving._generate_audio_sse_events(
|
||||
_gen(_result(torch.empty(0))),
|
||||
request_id="req-empty",
|
||||
response_format="pcm",
|
||||
)
|
||||
collected = await _collect(events)
|
||||
assert any("speech.audio.error" in event for event in collected)
|
||||
assert not any("speech.audio.done" in event for event in collected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_audex_empty_chunks_keep_legacy_behavior():
|
||||
"""Other models keep their existing semantics (no new rejection)."""
|
||||
serving = _serving(model_type="qwen3_tts")
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_gen(_result(torch.empty(0))),
|
||||
request_id="req-other",
|
||||
response_format="pcm",
|
||||
)
|
||||
await _collect(chunks) # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- CFG serving surface
|
||||
|
||||
import re # noqa: E402
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest # noqa: E402
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.audex import AudexAdapter # noqa: E402
|
||||
|
||||
|
||||
def _adapter() -> AudexAdapter:
|
||||
return AudexAdapter.__new__(AudexAdapter)
|
||||
|
||||
|
||||
def _speech_request(**extra_params) -> OpenAICreateSpeechRequest:
|
||||
payload = {"model": "audex", "input": "Hello world."}
|
||||
if extra_params:
|
||||
payload["extra_params"] = extra_params
|
||||
return OpenAICreateSpeechRequest(**payload)
|
||||
|
||||
|
||||
class _MarkerTokenizer:
|
||||
"""Special markers count one token; words and punctuation split."""
|
||||
|
||||
_TOKEN_RE = re.compile(r"<unk>|<\|[^|]*\|>|<[a-z_]+>|\w+|[^\w\s]")
|
||||
|
||||
def encode(self, text: str) -> list[int]:
|
||||
return [0] * len(self._TOKEN_RE.findall(text))
|
||||
|
||||
|
||||
class TestAudexCfgValidation:
|
||||
def test_no_cfg_scale_accepted(self):
|
||||
assert _adapter().validate(_speech_request()) is None
|
||||
|
||||
@pytest.mark.parametrize("scale", [1.0, 1.5, 3.0, 10.0])
|
||||
def test_in_range_scale_accepted(self, scale):
|
||||
assert _adapter().validate(_speech_request(cfg_scale=scale)) is None
|
||||
|
||||
@pytest.mark.parametrize("scale", [0.5, 0.0, -1.0, 10.5, "big"])
|
||||
def test_out_of_range_or_malformed_scale_rejected(self, scale):
|
||||
error = _adapter().validate(_speech_request(cfg_scale=scale))
|
||||
assert error is not None and "cfg_scale" in error
|
||||
|
||||
def test_explicit_null_scale_treated_as_absent(self):
|
||||
assert _adapter().validate(_speech_request(cfg_scale=None)) is None
|
||||
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": None})
|
||||
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": "p"}, params)
|
||||
assert out.extra_args == {}
|
||||
|
||||
@pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt"])
|
||||
def test_internal_cfg_keys_rejected(self, key):
|
||||
error = _adapter().validate(_speech_request(**{key: "x"}))
|
||||
assert error is not None and key in error
|
||||
|
||||
|
||||
class TestAudexCfgInjection:
|
||||
def _serving_with_fake_tokenizer(self):
|
||||
serving = _serving()
|
||||
serving._get_audex_tokenizer = lambda: _MarkerTokenizer()
|
||||
return serving
|
||||
|
||||
def _cond_prompt(self, text: str = "Hello world.") -> str:
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
return build_cond_prompt(text)
|
||||
|
||||
def test_no_extra_args_is_untouched(self):
|
||||
serving = self._serving_with_fake_tokenizer()
|
||||
params = SimpleNamespace(extra_args=None)
|
||||
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params)
|
||||
assert out.extra_args is None
|
||||
assert params.extra_args is None
|
||||
|
||||
def test_scale_one_leaves_params_byte_identical(self):
|
||||
serving = self._serving_with_fake_tokenizer()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.0})
|
||||
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": self._cond_prompt()}, params)
|
||||
assert out.extra_args == {}
|
||||
|
||||
def test_guided_request_gains_pair_contract(self):
|
||||
serving = self._serving_with_fake_tokenizer()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.5})
|
||||
cond = self._cond_prompt("Some transcription text here.")
|
||||
out = serving._inject_audex_cfg_pair_args("req-42", {"prompt": cond}, params)
|
||||
|
||||
extra = out.extra_args
|
||||
assert extra["cfg_scale"] == 1.5
|
||||
assert extra["cfg_role"] == "cond"
|
||||
assert extra["cfg_pair_id"] == "req-42"
|
||||
null_prompt = extra["cfg_null_prompt"]
|
||||
assert "<unk>" in null_prompt
|
||||
assert "Some transcription" not in null_prompt
|
||||
tok = _MarkerTokenizer()
|
||||
assert len(tok.encode(null_prompt)) == len(tok.encode(cond))
|
||||
|
||||
def test_missing_text_prompt_raises(self):
|
||||
serving = self._serving_with_fake_tokenizer()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.5})
|
||||
with pytest.raises(ValueError, match="adapter-built text prompt"):
|
||||
serving._inject_audex_cfg_pair_args("req-1", {"prompt_token_ids": [1]}, params)
|
||||
|
||||
def test_injection_never_mutates_input_params(self):
|
||||
"""The input may be the engine's SHARED default params (review P2):
|
||||
pair state must land only on the returned clone."""
|
||||
serving = self._serving_with_fake_tokenizer()
|
||||
shared = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
|
||||
out = serving._inject_audex_cfg_pair_args("req-7", {"prompt": self._cond_prompt()}, shared)
|
||||
assert out is not shared
|
||||
assert shared.extra_args == {"cfg_scale": 1.5}
|
||||
assert shared.temperature == 0.1
|
||||
assert out.extra_args["cfg_pair_id"] == "req-7"
|
||||
assert out.temperature == 0.05
|
||||
|
||||
|
||||
class TestAudexStageDetection:
|
||||
"""audex_omni is speech-capable only alongside the speech decoder."""
|
||||
|
||||
def _serving_with_stages(self, stages: list[str]) -> OmniOpenAIServingSpeech:
|
||||
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
serving.engine_client = SimpleNamespace(
|
||||
stage_configs=[SimpleNamespace(engine_args=SimpleNamespace(model_stage=s)) for s in stages]
|
||||
)
|
||||
return serving
|
||||
|
||||
def test_full_pipeline_detected_as_tts(self):
|
||||
serving = self._serving_with_stages(["audex_omni", "audex_code2wav"])
|
||||
stage = serving._find_tts_stage()
|
||||
assert stage is not None and stage.engine_args.model_stage == "audex_omni"
|
||||
|
||||
def test_thinker_only_not_detected_as_tts(self):
|
||||
serving = self._serving_with_stages(["audex_omni"])
|
||||
assert serving._find_tts_stage() is None
|
||||
|
||||
def test_tts_pipeline_still_detected(self):
|
||||
serving = self._serving_with_stages(["audex_thinker", "audex_code2wav"])
|
||||
stage = serving._find_tts_stage()
|
||||
assert stage is not None and stage.engine_args.model_stage == "audex_thinker"
|
||||
|
||||
|
||||
def test_guided_injection_sets_guided_temperature():
|
||||
"""Guided requests decode at the measured CFG temperature (0.05)."""
|
||||
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
|
||||
cond = TestAudexCfgInjection()._cond_prompt()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
|
||||
out = serving._inject_audex_cfg_pair_args("req-1", {"prompt": cond}, params)
|
||||
assert out.temperature == 0.05
|
||||
|
||||
unguided = SimpleNamespace(extra_args={"cfg_scale": 1.0}, temperature=0.1)
|
||||
out = serving._inject_audex_cfg_pair_args("req-2", {"prompt": cond}, unguided)
|
||||
assert out.temperature == 0.1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TTA serving surface
|
||||
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter # noqa: E402
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.audex_tta import AudexTTAAdapter # noqa: E402
|
||||
|
||||
|
||||
def _tta_adapter() -> AudexTTAAdapter:
|
||||
return AudexTTAAdapter.__new__(AudexTTAAdapter)
|
||||
|
||||
|
||||
class _TTAVocabTokenizer(_MarkerTokenizer):
|
||||
"""Marker tokenizer + a full audiocodec vocab for phase-id building."""
|
||||
|
||||
def get_vocab(self):
|
||||
vocab = {"<audiogen_start>": 7, "<audiogen_end>": 8}
|
||||
for n in range(8192):
|
||||
vocab[f"<audiocodec_{n}>"] = 1000 + n
|
||||
return vocab
|
||||
|
||||
|
||||
class TestAudexTTAAdapterValidation:
|
||||
def test_default_caption_accepted(self):
|
||||
assert _tta_adapter().validate(_speech_request()) is None
|
||||
|
||||
def test_empty_caption_rejected(self):
|
||||
req = _speech_request()
|
||||
req.input = " "
|
||||
assert _tta_adapter().validate(req) is not None
|
||||
|
||||
@pytest.mark.parametrize("scale", [1.0, 3.0, 10.0])
|
||||
def test_in_range_scale_accepted(self, scale):
|
||||
assert _tta_adapter().validate(_speech_request(cfg_scale=scale)) is None
|
||||
|
||||
@pytest.mark.parametrize("scale", [0.5, 10.5, "big"])
|
||||
def test_out_of_range_scale_rejected(self, scale):
|
||||
error = _tta_adapter().validate(_speech_request(cfg_scale=scale))
|
||||
assert error is not None and "cfg_scale" in error
|
||||
|
||||
@pytest.mark.parametrize("key", ["cfg_role", "cfg_pair_id", "cfg_null_prompt", "tta_rvq"])
|
||||
def test_internal_keys_rejected(self, key):
|
||||
error = _tta_adapter().validate(_speech_request(**{key: "x"}))
|
||||
assert error is not None and key in error
|
||||
|
||||
|
||||
class TestAdapterRouting:
|
||||
def test_tta_adapter_resolves_by_model_type(self):
|
||||
adapter_cls = resolve_adapter("audex_tta")
|
||||
assert adapter_cls is AudexTTAAdapter
|
||||
assert adapter_cls.stage_keys == frozenset({"audex_tta_thinker"})
|
||||
|
||||
def test_tts_adapter_never_selects_tta_stage(self):
|
||||
assert "audex_tta_thinker" not in AudexAdapter.stage_keys
|
||||
assert not (AudexAdapter.stage_keys & AudexTTAAdapter.stage_keys)
|
||||
|
||||
def test_detection_separates_tts_and_tta(self):
|
||||
tta_serving = TestAudexStageDetection()._serving_with_stages(["audex_tta_thinker", "audex_xcodec"])
|
||||
tta_serving._tts_stage = tta_serving._find_tts_stage()
|
||||
assert tta_serving._tts_stage.engine_args.model_stage == "audex_tta_thinker"
|
||||
assert tta_serving._detect_tts_model_type() == "audex_tta"
|
||||
|
||||
tts_serving = TestAudexStageDetection()._serving_with_stages(["audex_thinker", "audex_code2wav"])
|
||||
tts_serving._tts_stage = tts_serving._find_tts_stage()
|
||||
assert tts_serving._detect_tts_model_type() == "audex"
|
||||
|
||||
|
||||
class TestAudexTTAInjection:
|
||||
def _serving(self):
|
||||
serving = _serving(model_type="audex_tta")
|
||||
serving._get_audex_tokenizer = lambda: _TTAVocabTokenizer()
|
||||
serving._audex_tta_rvq = None
|
||||
return serving
|
||||
|
||||
def _prompt(self):
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt
|
||||
|
||||
return {"prompt": build_tta_cond_prompt("Rain on a tin roof.")}
|
||||
|
||||
def test_default_injection_applies_rvq_and_cfg3(self):
|
||||
serving = self._serving()
|
||||
params = SimpleNamespace(extra_args=None)
|
||||
out = serving._inject_audex_tta_args("req-9", self._prompt(), params)
|
||||
|
||||
extra = out.extra_args
|
||||
rvq = extra["tta_rvq"]
|
||||
assert len(rvq["phase_token_ids"]) == 4
|
||||
assert all(len(ids) == 1024 for ids in rvq["phase_token_ids"])
|
||||
assert (rvq["start_tid"], rvq["end_tid"]) == (7, 8)
|
||||
assert rvq["codec_cap"] == 4000 and rvq["start_in_prompt"] is True
|
||||
assert extra["cfg_scale"] == 3.0
|
||||
assert extra["cfg_role"] == "cond" and extra["cfg_pair_id"] == "req-9"
|
||||
assert "<unk>" in extra["cfg_null_prompt"]
|
||||
|
||||
def test_explicit_scale_one_disables_cfg_but_keeps_rvq(self):
|
||||
serving = self._serving()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.0})
|
||||
out = serving._inject_audex_tta_args("req-1", self._prompt(), params)
|
||||
assert "tta_rvq" in out.extra_args
|
||||
assert "cfg_scale" not in out.extra_args
|
||||
assert "cfg_role" not in out.extra_args
|
||||
|
||||
def test_rvq_contract_cached_per_process(self):
|
||||
serving = self._serving()
|
||||
out_a = serving._inject_audex_tta_args("a", self._prompt(), SimpleNamespace(extra_args=None))
|
||||
out_b = serving._inject_audex_tta_args("b", self._prompt(), SimpleNamespace(extra_args=None))
|
||||
assert out_a.extra_args["tta_rvq"] is out_b.extra_args["tta_rvq"]
|
||||
|
||||
def test_injection_never_mutates_input_params(self):
|
||||
"""The default online path hands the injector an element of the
|
||||
engine's SHARED default_sampling_params_list (review P2): the
|
||||
per-request pair state must land only on the returned clone."""
|
||||
serving = self._serving()
|
||||
shared = SimpleNamespace(extra_args=None)
|
||||
out_1 = serving._inject_audex_tta_args("req-1", self._prompt(), shared)
|
||||
out_2 = serving._inject_audex_tta_args("req-2", self._prompt(), shared)
|
||||
assert shared.extra_args is None
|
||||
assert out_1 is not shared and out_2 is not shared
|
||||
assert out_1.extra_args["cfg_pair_id"] == "req-1"
|
||||
assert out_2.extra_args["cfg_pair_id"] == "req-2"
|
||||
@@ -529,6 +529,26 @@ def decode_b64_image(b64: str):
|
||||
return img
|
||||
|
||||
|
||||
def concat_audio(audio_val) -> np.ndarray:
|
||||
"""Flatten a multimodal audio payload to mono float32 samples.
|
||||
|
||||
Engines return ``multimodal_output["audio"]`` as a tensor, a list of
|
||||
per-chunk tensors (streaming decoders), or an array-like; concatenate
|
||||
in order and return a 1-D ``np.float32`` array (empty when a list has
|
||||
no tensors).
|
||||
"""
|
||||
import torch
|
||||
|
||||
if isinstance(audio_val, list):
|
||||
tensors = [t.detach().cpu().float().reshape(-1) for t in audio_val if isinstance(t, torch.Tensor)]
|
||||
if not tensors:
|
||||
return np.zeros((0,), dtype=np.float32)
|
||||
return torch.cat(tensors, dim=-1).numpy().astype(np.float32, copy=False)
|
||||
if isinstance(audio_val, torch.Tensor):
|
||||
return audio_val.detach().cpu().float().reshape(-1).numpy()
|
||||
return np.asarray(audio_val, dtype=np.float32).reshape(-1)
|
||||
|
||||
|
||||
def preprocess_text(text):
|
||||
import opencc
|
||||
|
||||
@@ -695,6 +715,7 @@ def convert_audio_bytes_to_text(raw_bytes: bytes, model_size: str = "small") ->
|
||||
|
||||
__all__ = [
|
||||
"_merge_base64_audio_to_segment",
|
||||
"concat_audio",
|
||||
"convert_audio_bytes_to_text",
|
||||
"convert_audio_file_to_text",
|
||||
"cosine_similarity_text",
|
||||
|
||||
Reference in new issue
Block a user