[New Model] support nemotron Audex (#4976)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
64 files changed
+7760
-39
No files matched your search
@@ -0,0 +1,262 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E test for the Audex cascaded speech-to-speech pipeline.
|
||||
|
||||
One ``audex_s2s`` deployment serves all three official
|
||||
passes. The routing contract is the hard gate here: text-modality passes
|
||||
(ASR, chat) finish at stage 0 and must never produce stage-1 audio; the
|
||||
audio-modality TTS pass streams through the causal speech decoder.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
ASR_PROMPT = (
|
||||
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
)
|
||||
|
||||
# Official chat-pass recipe (examples/offline_inference/audex/speech_to_speech.py,
|
||||
# mirroring the official cascaded web demo) — 30B-A3B ONLY. The
|
||||
# formatting-instruction prefix is load-bearing on the 30B-A3B: without it a bare
|
||||
# transcript turn is treated as a TTS request and the model emits literal
|
||||
# <speechcodec_N> tokens instead of text, and under the yaml's near-greedy
|
||||
# defaults its reasoning block never closes. The 2B must NOT use this recipe:
|
||||
# it is long-verified stable on a bare prompt + engine-default sampling, while
|
||||
# the official temp-1.0 seed-0 draw hallucinates off-language text (yet closes
|
||||
# </think> cleanly, so the seed retry never fires) and fails the round-trip
|
||||
# WER gate — measured, reproduced byte-identically.
|
||||
SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
|
||||
TEXT_INSTRUCTION = (
|
||||
"SYSTEM & FORMATTING INSTRUCTION: You are Audex, created by NVIDIA based on the Nemotron-Cascade-2 architecture. "
|
||||
"You may output your reasoning, followed by your final response. "
|
||||
"You may format your reasoning block however you like. "
|
||||
"However, your final response must strictly follow these rules: "
|
||||
"* Write in plain, unformatted prose like a book or newspaper article. "
|
||||
"* Do not use markdown, bullet points, lists, or headers. "
|
||||
"* Standard numbers, abbreviations, and symbols are acceptable. "
|
||||
"* [CRITICAL] You must press enter after every single sentence, placing each sentence on its own separate line."
|
||||
)
|
||||
CHAT_MAX_TOKENS = 2048 # official --text-max-tokens default
|
||||
# The official demo samples this pass unseeded; some trajectories deliberate
|
||||
# past the token budget without ever closing the think block, so retry a few
|
||||
# deterministic seeds until one closes.
|
||||
CHAT_SEEDS = (0, 1, 2)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_s2s.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": True}),
|
||||
id="s2s_full",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_s2s_30b.yaml"), {"async_chunk": True}),
|
||||
id="s2s_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(text: str) -> list[str]:
|
||||
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
|
||||
|
||||
|
||||
def _wer(hyp: str, ref: str) -> float:
|
||||
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
|
||||
if not ref_words:
|
||||
return 0.0
|
||||
rows = list(range(len(hyp_words) + 1))
|
||||
for i, ref_word in enumerate(ref_words, 1):
|
||||
prev, rows[0] = rows[0], i
|
||||
for j, hyp_word in enumerate(hyp_words, 1):
|
||||
cur = min(rows[j] + 1, rows[j - 1] + 1, prev + (ref_word != hyp_word))
|
||||
prev, rows[j] = rows[j], cur
|
||||
return rows[-1] / len(ref_words)
|
||||
|
||||
|
||||
def _chat_prompt(user_text: str) -> str:
|
||||
# Official chat-pass recipe: system prompt + formatting instruction
|
||||
# prefixed to the user text, thinking ENABLED via an open <think> block
|
||||
# (chat_template.jinja with enable_thinking=True renders
|
||||
# "<|im_start|>assistant\n<think>\n"). Priming the block open keeps the
|
||||
# reasoning inside <think>...</think> so _extract_final_response can drop
|
||||
# it; with a closed block the 30B leaks the reasoning as plain text.
|
||||
return (
|
||||
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{TEXT_INSTRUCTION}\n\n{user_text}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n<think>\n"
|
||||
)
|
||||
|
||||
|
||||
def _extract_final_response(text: str) -> str:
|
||||
# Mirror the official demo's extract_spoken_answer: keep only what
|
||||
# follows the LAST closing think tag, falling back to the full text when
|
||||
# the model skipped reasoning or never closed the block. Benign no-op for
|
||||
# the 2B, whose answers carry no think block.
|
||||
if "</think>" in text:
|
||||
answer = text.rsplit("</think>", 1)[-1].strip()
|
||||
if answer:
|
||||
return answer
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _chat_params(runner: OmniRunner, seed: int):
|
||||
# Official demo text-pass sampling (--text-temperature 1.0, --text-top-p
|
||||
# 0.95, --text-max-tokens 2048). The yaml stage-0 defaults (temperature
|
||||
# 0.1, top_k 80) are TTS-thinker settings; under near-greedy decoding the
|
||||
# 30B loops inside its reasoning block until the token budget runs out
|
||||
# and never emits </think>.
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
stage0.temperature = 1.0
|
||||
stage0.top_p = 0.95
|
||||
stage0.top_k = -1
|
||||
stage0.max_tokens = CHAT_MAX_TOKENS
|
||||
stage0.seed = seed
|
||||
return params
|
||||
|
||||
|
||||
def _chat_once(runner: OmniRunner, prompt: str, seed: int) -> tuple[str, bool]:
|
||||
(req_output,) = runner.omni.generate([{"prompt": prompt, "modalities": ["text"]}], _chat_params(runner, seed))
|
||||
out = req_output.outputs[0]
|
||||
text = (out.text or "").strip()
|
||||
truncated = getattr(out, "finish_reason", None) == "length" or len(out.token_ids or []) >= CHAT_MAX_TOKENS
|
||||
return text, truncated
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_audex_s2s_cascade_round_trip(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""ASR → chat → TTS over one deployment; text passes never emit audio."""
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
# Spoken question: vLLM's public audio asset (downloaded on first use).
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
# Pass 1 — ASR (text-final).
|
||||
asr_outputs = omni_runner.omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": ASR_PROMPT,
|
||||
"multi_modal_data": {"audio": (audio, sr)},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
]
|
||||
)
|
||||
assert len(asr_outputs) == 1
|
||||
asr_out = asr_outputs[0]
|
||||
transcript = (asr_out.outputs[0].text or "").strip()
|
||||
mm = getattr(asr_out, "multimodal_output", None) or {}
|
||||
stage1_audio = concat_audio(mm.get("audio")) if "audio" in mm else np.zeros((0,), dtype=np.float32)
|
||||
assert stage1_audio.size == 0, "Text-modality ASR pass produced stage-1 audio (routing regression)"
|
||||
|
||||
# Pass 2 — chat (text-final). The recipe is size-conditional (see the
|
||||
# SYSTEM_PROMPT comment above): the 30B requires the official recipe,
|
||||
# the 2B regresses under it.
|
||||
if omni_runner.model_name == _AUDEX_30B_MODEL:
|
||||
# Official recipe. The raw completion is
|
||||
# <reasoning...></think><answer>; speak only the final response.
|
||||
# Retry with the next seed only when the budget ran out before the
|
||||
# reasoning block closed (a natural stop without </think> is a
|
||||
# complete answer); if every seed spins, keep the full text
|
||||
# (official demo behavior).
|
||||
chat_prompt = _chat_prompt(transcript or "Hello.")
|
||||
raw = ""
|
||||
for seed in CHAT_SEEDS:
|
||||
raw, truncated = _chat_once(omni_runner, chat_prompt, seed)
|
||||
if "</think>" in raw or not truncated:
|
||||
break
|
||||
answer = _extract_final_response(raw)
|
||||
else:
|
||||
# 2B: bare prompt with a closed think block, engine-default sampling
|
||||
# (yaml near-greedy), no retry; the answer is used as-is.
|
||||
chat_prompt = f"<|im_start|>user\n{transcript or 'Hello.'}<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
(chat_output,) = omni_runner.omni.generate([{"prompt": chat_prompt, "modalities": ["text"]}])
|
||||
answer = (chat_output.outputs[0].text or "").strip()
|
||||
|
||||
if real_weights:
|
||||
assert transcript, "ASR pass produced an empty transcript"
|
||||
assert answer, "Chat pass produced an empty answer"
|
||||
|
||||
# Pass 3 — TTS (audio-final; streams through code2wav). Both final
|
||||
# stages may emit an output record; collect the audio-bearing one.
|
||||
# The formatting instruction puts every sentence on its own line;
|
||||
# flatten to plain prose for the TTS thinker.
|
||||
tts_text = re.sub(r"\s+", " ", answer).strip() if answer else "The weather is nice today."
|
||||
tts_text = tts_text[:200]
|
||||
tts_prompt = {"prompt": build_cond_prompt(tts_text), "modalities": ["audio"]}
|
||||
tts_outputs = omni_runner.omni.generate([tts_prompt])
|
||||
assert tts_outputs, "TTS pass returned no outputs"
|
||||
speech = np.zeros((0,), dtype=np.float32)
|
||||
for req_output in tts_outputs:
|
||||
mm = getattr(req_output, "multimodal_output", None) or {}
|
||||
if "audio" in mm:
|
||||
speech = concat_audio(mm["audio"])
|
||||
break
|
||||
assert speech.size > 0, "TTS pass produced empty audio"
|
||||
|
||||
if real_weights:
|
||||
rms = float(np.sqrt(np.mean(np.square(speech))))
|
||||
assert rms > 1e-3, f"TTS-pass audio near-silent (rms={rms})"
|
||||
|
||||
# Pass 4 — output verification: transcribe the synthesized answer
|
||||
# with the SAME deployment's ASR pass and compare to the answer
|
||||
# text (self-contained; no external ASR dependency).
|
||||
verify_outputs = omni_runner.omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": ASR_PROMPT,
|
||||
"multi_modal_data": {"audio": (speech, SAMPLE_RATE)},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
]
|
||||
)
|
||||
verify_text = (verify_outputs[0].outputs[0].text or "").strip()
|
||||
assert verify_text, "Output-verification ASR pass returned empty text"
|
||||
# The full checkpoint prefixes transcripts with a language-tag
|
||||
# sentence and quotes the content; extract the quoted payload.
|
||||
if "'" in verify_text and len(verify_text.split("'")) >= 3:
|
||||
verify_text = verify_text.split("'")[1].strip()
|
||||
wer = _wer(verify_text, tts_text)
|
||||
# Hard gate: the synthesized WAV must transcribe back to the answer
|
||||
# (lenient threshold absorbs ASR quoting/prefix differences).
|
||||
assert wer <= 0.5, f"Output WAV does not match the answer (WER={wer:.2f}): {verify_text!r} vs {tts_text!r}"
|
||||
print(f"[trend] Audex S2S output-ASR WER vs answer: {wer:.3f} ({verify_text!r})")
|
||||
|
||||
# Final pass — time-to-first-audio for the streaming TTS path. This
|
||||
# MUST be the last engine call: the py_generator API closes the engine
|
||||
# when the generator is exhausted.
|
||||
t_start = time.perf_counter()
|
||||
first_audio_s: float | None = None
|
||||
for req_output in omni_runner.omni.generate([tts_prompt], py_generator=True):
|
||||
mm = getattr(req_output, "multimodal_output", None) or {}
|
||||
if "audio" in mm and concat_audio(mm["audio"]).size > 0:
|
||||
if first_audio_s is None:
|
||||
first_audio_s = time.perf_counter() - t_start
|
||||
assert first_audio_s is not None, "streaming TTS never produced an audio-bearing chunk"
|
||||
print(f"[trend] Audex S2S TTS-pass first-audio latency: {first_audio_s * 1000.0:.0f} ms")
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex audio understanding (thinker-only).
|
||||
|
||||
Single-stage ``audex_thinker_only`` pipeline on
|
||||
``checkpoint_folder_full``: WAV (+ instruction) in, text out. The input clip
|
||||
is vLLM's public ``mary_had_lamb`` audio asset (downloaded on first use), so
|
||||
its transcript is known.
|
||||
|
||||
WER is a trend metric (recorded in the assertion message, no numeric hard
|
||||
gate); the hard gates are structural: non-empty, non-degenerate text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
|
||||
# Full transcript of the mary_had_lamb asset (Edison's phonograph recording);
|
||||
# a shortened reference would count the extra spoken words as insertions and
|
||||
# inflate WER past the trend threshold.
|
||||
_REFERENCE = (
|
||||
"the first words i spoke in the original phonograph a little piece of practical poetry "
|
||||
"mary had a little lamb its fleece was white as snow and everywhere that mary went the "
|
||||
"lamb was sure to go"
|
||||
)
|
||||
|
||||
ASR_PROMPT = (
|
||||
"<|im_start|>user\n<so_embedding>\nTranscribe the input speech.<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_thinker_only.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": False}),
|
||||
id="thinker_only",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_thinker_only_30b.yaml"), {"async_chunk": False}),
|
||||
id="thinker_only_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(text: str) -> list[str]:
|
||||
return "".join(c.lower() if c.isalnum() or c.isspace() else " " for c in text).split()
|
||||
|
||||
|
||||
def _wer(hyp: str, ref: str) -> float:
|
||||
hyp_words, ref_words = _normalize(hyp), _normalize(ref)
|
||||
if not ref_words:
|
||||
return 0.0
|
||||
rows = list(range(len(hyp_words) + 1))
|
||||
for i, ref_word in enumerate(ref_words, 1):
|
||||
prev, rows[0] = rows[0], i
|
||||
for j, hyp_word in enumerate(hyp_words, 1):
|
||||
cur = min(
|
||||
rows[j] + 1,
|
||||
rows[j - 1] + 1,
|
||||
prev + (ref_word != hyp_word),
|
||||
)
|
||||
prev, rows[j] = rows[j], cur
|
||||
return rows[-1] / len(ref_words)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_asr_smoke(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""Transcribing the public audio asset yields coherent, non-degenerate text."""
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
outputs = omni_runner.omni.generate([{"prompt": ASR_PROMPT, "multi_modal_data": {"audio": (audio, sr)}}])
|
||||
|
||||
assert len(outputs) == 1
|
||||
text = outputs[0].outputs[0].text if outputs[0].outputs else ""
|
||||
assert isinstance(text, str)
|
||||
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
words = _normalize(text)
|
||||
assert words, f"Empty transcription: {text!r}"
|
||||
# Degenerate repetition guard: no word may dominate the output.
|
||||
most_common = max(words.count(w) for w in set(words))
|
||||
assert most_common <= max(3, len(words) // 2), f"Degenerate transcript: {text!r}"
|
||||
wer = _wer(text, _REFERENCE)
|
||||
# Trend metric (recorded, not gated): substantially-correct check only.
|
||||
assert wer <= 0.9, f"Transcript unrelated to reference (WER={wer:.2f}): {text!r}"
|
||||
print(f"[trend] Audex S2T WER vs reference: {wer:.3f} ({text!r})")
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex text-to-audio (TTA).
|
||||
|
||||
Caption → RVQ-phase-masked <audiocodec_*> tokens → XCodec1 waveform through
|
||||
the ``audex_tta`` pipeline. RVQ phase validity is enforced by
|
||||
the mask logits processor (unit-tested against the official validator); the
|
||||
hard gates here are decode-side: non-silent, finite audio at the documented
|
||||
sample rate. CFG runs at the official TTA default (3.0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
# The acceptance gate requires eight captions decoding to non-silent,
|
||||
# finite audio through XCodec1.
|
||||
CAPTIONS = (
|
||||
"Heavy rain falling on a tin roof.",
|
||||
"A dog barking in the distance while birds chirp.",
|
||||
"Ocean waves crashing on a rocky shore.",
|
||||
"A crackling campfire at night with crickets.",
|
||||
"Thunder rumbling during a storm.",
|
||||
"A steam train whistle blowing as it departs.",
|
||||
"Wind chimes ringing in a gentle breeze.",
|
||||
"Footsteps crunching through fresh snow.",
|
||||
)
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_tta.yaml")
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": False}),
|
||||
id="tta_full_payload",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tta_30b.yaml"), {"async_chunk": False}),
|
||||
id="tta_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tta_tokenizer_bits():
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids
|
||||
|
||||
root = ensure_audex_snapshot(_audex_model, profile="tta")
|
||||
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
|
||||
phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer)
|
||||
return tokenizer, phase_token_ids, start_tid, end_tid
|
||||
|
||||
|
||||
def _tta_sampling_params(runner: OmniRunner, cond_prompt: str, tokenizer_bits, cfg_scale: float = 3.0):
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt
|
||||
|
||||
tokenizer, phase_token_ids, start_tid, end_tid = tokenizer_bits
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
if stage0.extra_args is None:
|
||||
stage0.extra_args = {}
|
||||
stage0.extra_args["tta_rvq"] = {
|
||||
"phase_token_ids": phase_token_ids,
|
||||
"start_tid": start_tid,
|
||||
"end_tid": end_tid,
|
||||
"codec_cap": 4000,
|
||||
"start_in_prompt": True,
|
||||
}
|
||||
if cfg_scale > 1.0:
|
||||
stage0.extra_args.update(
|
||||
{
|
||||
"cfg_scale": float(cfg_scale),
|
||||
"cfg_role": "cond",
|
||||
"cfg_pair_id": f"e2e-tta-{abs(hash(cond_prompt)) % 10_000_000}",
|
||||
"cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer),
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_tta_eight_captions(omni_runner: OmniRunner, run_level: str, tta_tokenizer_bits) -> None:
|
||||
"""All eight captions decode through XCodec1 to non-silent, finite audio.
|
||||
|
||||
RVQ phase validity is enforced by the mask logits processor by
|
||||
construction; the final user output carries only decoded audio (the
|
||||
audio-final pipeline emits no stage-0 token record), and disabling the
|
||||
mask in-engine as an e2e control would require a second engine boot
|
||||
with a different logits-processor config. The mask's sensitivity is
|
||||
therefore proven at unit level (see test_audex_tta.py's mask tests and
|
||||
the unmasked-sampling control) against the official validator.
|
||||
"""
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
for caption in CAPTIONS:
|
||||
prompt = build_tta_cond_prompt(caption)
|
||||
params = _tta_sampling_params(omni_runner, prompt, tta_tokenizer_bits)
|
||||
outputs = omni_runner.omni.generate([prompt], params)
|
||||
|
||||
assert len(outputs) == 1, f"{caption!r}: expected one output, got {len(outputs)}"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, f"{caption!r}: no audio output: {list(audio_mm.keys())}"
|
||||
audio = concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, f"{caption!r}: generated audio is empty"
|
||||
assert np.isfinite(audio).all(), f"{caption!r}: audio contains NaN/Inf"
|
||||
|
||||
sr_val = audio_mm.get("sr", SAMPLE_RATE)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
assert int(sr_val) == SAMPLE_RATE, f"{caption!r}: unexpected sample_rate={sr_val}"
|
||||
|
||||
if real_weights:
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
# The decode cap is 500 frames = 10 s at 50 fps.
|
||||
assert 0.1 <= duration_s <= 10.5, f"{caption!r}: unexpected duration={duration_s:.3f}s"
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-4, f"{caption!r}: audio is near-silent (rms={rms})"
|
||||
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline E2E smoke test for Audex (Nemotron-Labs-Audex-2B) TTS.
|
||||
|
||||
Passes the HF repo ROOT and verifies the 2-stage pipeline (thinker →
|
||||
streaming causal speech decoder): root-path/subfolder resolution, codec-token
|
||||
extraction, per-request streaming sessions, and the lookahead flush at EOS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import concat_audio
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
SYNTH_TEXTS = (
|
||||
"The weather is so good, and I want to enjoy the beautiful morning in the park.",
|
||||
"The quick brown fox jumps over the lazy dog.",
|
||||
)
|
||||
|
||||
|
||||
_audex_deployment = get_deploy_config_path("audex_tts.yaml")
|
||||
# Collection must not download anything: pass the repo id (or a local
|
||||
# override) and let the engine's stage-init path resolve/download the
|
||||
# required snapshot subset at execution time (ensure_audex_snapshot).
|
||||
_audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_audex_model, _audex_deployment, {"async_chunk": True}),
|
||||
id="async_chunk",
|
||||
),
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
_OMNI_RUNNER_PARAMS.append(
|
||||
pytest.param(
|
||||
(_AUDEX_30B_MODEL, get_deploy_config_path("audex_tts_30b.yaml"), {"async_chunk": True}),
|
||||
id="tts_30b",
|
||||
)
|
||||
)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_tts_smoke(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""Audex TTS from the repo root should produce sane 16 kHz audio per prompt.
|
||||
|
||||
At ``core_model`` level the runner loads DUMMY weights (structural smoke
|
||||
only); real-speech assertions apply from ``advanced_model`` up.
|
||||
"""
|
||||
prompts = [build_cond_prompt(text) for text in SYNTH_TEXTS]
|
||||
outputs = omni_runner.omni.generate(prompts)
|
||||
|
||||
real_weights = run_level in {"advanced_model", "full_model"}
|
||||
assert len(outputs) == len(SYNTH_TEXTS), f"expected {len(SYNTH_TEXTS)} outputs, got {len(outputs)}"
|
||||
for output in outputs:
|
||||
audio_mm = output.multimodal_output
|
||||
assert "audio" in audio_mm, f"No audio output found: {list(audio_mm.keys())}"
|
||||
audio = concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
|
||||
sr_val = audio_mm.get("sr", SAMPLE_RATE)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
assert int(sr_val) == SAMPLE_RATE, f"Unexpected sample_rate={sr_val}"
|
||||
|
||||
if real_weights:
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
|
||||
# Real speech, not silence.
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-3, f"Audio is near-silent (rms={rms})"
|
||||
|
||||
|
||||
def _cfg_sampling_params(runner: OmniRunner, cfg_scale: float, pair_id: str, cond_prompt: str):
|
||||
"""Stage sampling params carrying the CFG pair contract for one request."""
|
||||
import copy
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_null_prompt
|
||||
|
||||
# Resolve the ACTIVE param's model (2B or 30B), not the module default:
|
||||
# the null prompt must be built with the same tokenizer the engine uses.
|
||||
root = ensure_audex_snapshot(runner.model_name)
|
||||
tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True)
|
||||
params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
if stage0.extra_args is None:
|
||||
stage0.extra_args = {}
|
||||
stage0.extra_args.update(
|
||||
{
|
||||
"cfg_scale": float(cfg_scale),
|
||||
"cfg_role": "cond",
|
||||
"cfg_pair_id": pair_id,
|
||||
"cfg_null_prompt": build_null_prompt(cond_prompt, tokenizer),
|
||||
}
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_cfg_guided_single_stream(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""A guided request must yield exactly ONE audio stream (companion suppressed)."""
|
||||
prompt = build_cond_prompt(SYNTH_TEXTS[0])
|
||||
params = _cfg_sampling_params(omni_runner, 1.5, "e2e-cfg-pair", prompt)
|
||||
outputs = omni_runner.omni.generate([prompt], params)
|
||||
|
||||
assert len(outputs) == 1, f"expected exactly one output (no companion leak), got {len(outputs)}"
|
||||
audio = concat_audio(outputs[0].multimodal_output["audio"])
|
||||
assert audio.size > 0, "Guided generation produced empty audio"
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-3, f"Guided audio is near-silent (rms={rms})"
|
||||
duration_s = audio.size / SAMPLE_RATE
|
||||
assert 0.5 <= duration_s <= 20.0, f"Unexpected duration={duration_s:.3f}s"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_audex_offline_cfg_disabled_is_byte_identical(omni_runner: OmniRunner, run_level: str) -> None:
|
||||
"""cfg_scale=1.0 must be indistinguishable from not passing cfg at all."""
|
||||
prompt = build_cond_prompt(SYNTH_TEXTS[1])
|
||||
|
||||
baseline = omni_runner.omni.generate([prompt])
|
||||
base_audio = concat_audio(baseline[0].multimodal_output["audio"])
|
||||
|
||||
import copy
|
||||
|
||||
params = copy.deepcopy(omni_runner.omni.resolve_sampling_params_list(None))
|
||||
if params[0].extra_args is None:
|
||||
params[0].extra_args = {}
|
||||
params[0].extra_args["cfg_scale"] = 1.0
|
||||
disabled = omni_runner.omni.generate([prompt], params)
|
||||
disabled_audio = concat_audio(disabled[0].multimodal_output["audio"])
|
||||
|
||||
assert base_audio.shape == disabled_audio.shape, (
|
||||
f"cfg_scale=1.0 changed output length: {base_audio.shape} vs {disabled_audio.shape}"
|
||||
)
|
||||
np.testing.assert_array_equal(base_audio, disabled_audio)
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) text-to-audio.
|
||||
|
||||
Covers /v1/audio/speech through the ``audex_tta`` deployment:
|
||||
the ``audex_tta`` adapter builds the caption prompt and the serving layer
|
||||
injects the RVQ phase contract plus the official default guidance
|
||||
(cfg_scale 3.0) server-side.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
CAPTION = "Heavy rain falling on a tin roof."
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
audex_tta_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="audex_tta",
|
||||
)
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
audex_tta_server_params.append(
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_AUDEX_30B_MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tta_30b.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="online_tta_30b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_tta_server_params, indirect=True)
|
||||
def test_audex_tta_online_smoke(omni_server, openai_client, run_level) -> None:
|
||||
"""A caption returns a valid finite non-silent WAV at the default cfg 3.0."""
|
||||
import io
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={"model": omni_server.model, "input": CAPTION, "response_format": "wav"},
|
||||
timeout=600,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text[:200]
|
||||
assert resp.content[:4] == b"RIFF", "not a WAV response"
|
||||
|
||||
with wave.open(io.BytesIO(resp.content)) as wav:
|
||||
assert wav.getframerate() == SAMPLE_RATE, f"unexpected sample rate {wav.getframerate()}"
|
||||
frames = wav.readframes(wav.getnframes())
|
||||
duration_s = wav.getnframes() / wav.getframerate()
|
||||
|
||||
audio = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
assert audio.size > 0, "empty audio body"
|
||||
assert np.isfinite(audio).all(), "audio contains NaN/Inf"
|
||||
|
||||
if run_level in {"advanced_model", "full_model"}:
|
||||
# The decode cap is 500 frames = 10 s at 50 fps.
|
||||
assert 0.1 <= duration_s <= 10.5, f"unexpected duration={duration_s:.3f}s"
|
||||
rms = float(np.sqrt(np.mean(np.square(audio))))
|
||||
assert rms > 1e-4, f"audio is near-silent (rms={rms})"
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E online tests for Audex (Nemotron-Labs-Audex-2B) TTS.
|
||||
|
||||
Covers /v1/audio/speech non-streaming and audio-byte streaming, plus the
|
||||
Audex request policies: single built-in voice (missing/empty/"default" only),
|
||||
no reference audio, and cfg_scale reserved at 1.0.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
MODEL = "nvidia/Nemotron-Labs-Audex-2B"
|
||||
PROMPT = "The weather is so good, and I want to enjoy the beautiful morning in the park."
|
||||
|
||||
audex_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tts.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="audex",
|
||||
)
|
||||
]
|
||||
# 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1.
|
||||
_AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B"
|
||||
if os.environ.get("AUDEX_E2E_30B") == "1":
|
||||
audex_server_params.append(
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_AUDEX_30B_MODEL,
|
||||
stage_config_path=get_deploy_config_path("audex_tts_30b.yaml"),
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="online_tts_30b",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_nonstream(omni_server, openai_client) -> None:
|
||||
"""Plain English TTS, non-streaming WAV response."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_stream_audio(omni_server, openai_client) -> None:
|
||||
"""Plain English TTS, raw audio-byte streaming."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_max_new_tokens_caps_duration(omni_server, openai_client, run_level) -> None:
|
||||
"""max_new_tokens must cap stage-0 generation (review P2 regression)."""
|
||||
import io
|
||||
import wave
|
||||
|
||||
import requests
|
||||
|
||||
if run_level not in {"advanced_model", "full_model"}:
|
||||
pytest.skip("duration cap needs real weights")
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"response_format": "wav",
|
||||
"max_new_tokens": 25,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
with wave.open(io.BytesIO(resp.content)) as wav:
|
||||
duration_s = wav.getnframes() / wav.getframerate()
|
||||
# 25 codec frames at 50 fps = 0.5 s (+ small lookahead tail); the full
|
||||
# sentence would be ~3 s, so a capped result proves the override applied.
|
||||
assert duration_s <= 1.0, f"max_new_tokens ignored: duration={duration_s:.2f}s"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_request_policies(omni_server, openai_client) -> None:
|
||||
"""Unsupported voice / CFG / empty input must be rejected with 400."""
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
|
||||
ok = requests.post(url, json={"model": omni_server.model, "input": PROMPT, "voice": "default"})
|
||||
assert ok.status_code == 200, ok.text
|
||||
|
||||
for bad_payload, needle in (
|
||||
({"input": PROMPT, "voice": "alloy"}, "voice"),
|
||||
({"input": " "}, "non-empty"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": 0.5}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": 10.5}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_scale": "big"}}, "cfg_scale"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_role": "cond"}}, "cfg_role"),
|
||||
({"input": PROMPT, "extra_params": {"cfg_pair_id": "x"}}, "cfg_pair_id"),
|
||||
({"input": PROMPT, "ref_audio": "https://example.com/ref.wav"}, "reference audio"),
|
||||
):
|
||||
resp = requests.post(url, json={"model": omni_server.model, **bad_payload})
|
||||
assert resp.status_code == 400, f"{bad_payload} -> {resp.status_code}: {resp.text[:200]}"
|
||||
assert needle in resp.text, f"{bad_payload} error missing {needle!r}: {resp.text[:200]}"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_cfg_nonstream(omni_server, openai_client) -> None:
|
||||
"""Guided (cfg_scale=1.5) non-streaming request returns one WAV."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_cfg_stream_audio(omni_server, openai_client) -> None:
|
||||
"""Guided (cfg_scale=1.5) streaming request yields a single audio stream."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_mixed_cfg_concurrency(omni_server, openai_client) -> None:
|
||||
"""Mixed guided/unguided requests at concurrency 4 all complete with audio."""
|
||||
import concurrent.futures
|
||||
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
payloads = [
|
||||
{"model": omni_server.model, "input": f"{PROMPT} Take {i}.", "response_format": "wav"}
|
||||
| ({"extra_params": {"cfg_scale": 1.5}} if i % 2 == 0 else {})
|
||||
for i in range(4)
|
||||
]
|
||||
|
||||
def _post(payload):
|
||||
return requests.post(url, json=payload, timeout=300)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
responses = list(pool.map(_post, payloads))
|
||||
|
||||
for payload, resp in zip(payloads, responses):
|
||||
assert resp.status_code == 200, f"{payload} -> {resp.status_code}: {resp.text[:200]}"
|
||||
assert resp.content[:4] == b"RIFF", f"{payload}: not a WAV response"
|
||||
assert len(resp.content) > 44, f"{payload}: empty audio body"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", audex_server_params, indirect=True)
|
||||
def test_audex_tts_stream_abort_leaves_server_healthy(omni_server, openai_client) -> None:
|
||||
"""Aborting a guided streaming request mid-flight must not wedge the server.
|
||||
|
||||
The abort cascades to the CFG companion (cfg_companion_tracker); a
|
||||
follow-up guided request on the same server must still succeed.
|
||||
"""
|
||||
import requests
|
||||
|
||||
url = f"{openai_client.base_url.rstrip('/')}/v1/audio/speech"
|
||||
long_input = " ".join([PROMPT] * 4)
|
||||
|
||||
with requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": long_input,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
},
|
||||
stream=True,
|
||||
timeout=300,
|
||||
) as resp:
|
||||
assert resp.status_code == 200, resp.text
|
||||
# Read one chunk, then abort the connection mid-stream.
|
||||
next(resp.iter_content(chunk_size=4096))
|
||||
|
||||
follow_up = requests.post(
|
||||
url,
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": PROMPT,
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5},
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
assert follow_up.status_code == 200, follow_up.text[:200]
|
||||
assert follow_up.content[:4] == b"RIFF" and len(follow_up.content) > 44
|
||||
Reference in new issue
Block a user