# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Offline E2E smoke test for Audex text-to-audio (TTA). Caption → RVQ-phase-masked tokens → XCodec1 waveform through the ``audex_tta`` pipeline. RVQ phase validity is enforced by the mask logits processor (unit-tested against the official validator); the hard gates here are decode-side: non-silent, finite audio at the documented sample rate. CFG runs at the official TTA default (3.0). """ from __future__ import annotations import copy import os import numpy as np import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import concat_audio from tests.helpers.runtime import OmniRunner from tests.helpers.stage_config import get_deploy_config_path from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt MODEL = "nvidia/Nemotron-Labs-Audex-2B" MODEL_DIR_ENV = "VLLM_OMNI_AUDEX_MODEL_DIR" SAMPLE_RATE = 16_000 # The acceptance gate requires eight captions decoding to non-silent, # finite audio through XCodec1. CAPTIONS = ( "Heavy rain falling on a tin roof.", "A dog barking in the distance while birds chirp.", "Ocean waves crashing on a rocky shore.", "A crackling campfire at night with crickets.", "Thunder rumbling during a storm.", "A steam train whistle blowing as it departs.", "Wind chimes ringing in a gentle breeze.", "Footsteps crunching through fresh snow.", ) _audex_deployment = get_deploy_config_path("audex_tta.yaml") _audex_model = os.environ.get(MODEL_DIR_ENV) or MODEL _OMNI_RUNNER_PARAMS = [ pytest.param( (_audex_model, _audex_deployment, {"async_chunk": False}), id="tta_full_payload", ), ] # 30B-A3B variant: opt-in (pulls ~60 GB); enable with AUDEX_E2E_30B=1. _AUDEX_30B_MODEL = os.environ.get("VLLM_OMNI_AUDEX_30B_MODEL_DIR") or "nvidia/Nemotron-Labs-Audex-30B-A3B" if os.environ.get("AUDEX_E2E_30B") == "1": _OMNI_RUNNER_PARAMS.append( pytest.param( (_AUDEX_30B_MODEL, get_deploy_config_path("audex_tta_30b.yaml"), {"async_chunk": False}), id="tta_30b", ) ) pytestmark = [ pytest.mark.slow, pytest.mark.tts, pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True), ] @pytest.fixture(scope="module") def tta_tokenizer_bits(): from transformers import AutoTokenizer from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids root = ensure_audex_snapshot(_audex_model, profile="tta") tokenizer = AutoTokenizer.from_pretrained(os.path.join(root, "checkpoint_folder_audiogen"), trust_remote_code=True) phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer) return tokenizer, phase_token_ids, start_tid, end_tid def _tta_sampling_params(runner: OmniRunner, cond_prompt: str, tokenizer_bits, cfg_scale: float = 3.0): from vllm_omni.model_executor.models.audex.prompt import build_tta_null_prompt tokenizer, phase_token_ids, start_tid, end_tid = tokenizer_bits params = copy.deepcopy(runner.omni.resolve_sampling_params_list(None)) stage0 = params[0] if stage0.extra_args is None: stage0.extra_args = {} stage0.extra_args["tta_rvq"] = { "phase_token_ids": phase_token_ids, "start_tid": start_tid, "end_tid": end_tid, "codec_cap": 4000, "start_in_prompt": True, } if cfg_scale > 1.0: stage0.extra_args.update( { "cfg_scale": float(cfg_scale), "cfg_role": "cond", "cfg_pair_id": f"e2e-tta-{abs(hash(cond_prompt)) % 10_000_000}", "cfg_null_prompt": build_tta_null_prompt(cond_prompt, tokenizer), } ) return params @pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_audex_offline_tta_eight_captions(omni_runner: OmniRunner, run_level: str, tta_tokenizer_bits) -> None: """All eight captions decode through XCodec1 to non-silent, finite audio. RVQ phase validity is enforced by the mask logits processor by construction; the final user output carries only decoded audio (the audio-final pipeline emits no stage-0 token record), and disabling the mask in-engine as an e2e control would require a second engine boot with a different logits-processor config. The mask's sensitivity is therefore proven at unit level (see test_audex_tta.py's mask tests and the unmasked-sampling control) against the official validator. """ real_weights = run_level in {"advanced_model", "full_model"} for caption in CAPTIONS: prompt = build_tta_cond_prompt(caption) params = _tta_sampling_params(omni_runner, prompt, tta_tokenizer_bits) outputs = omni_runner.omni.generate([prompt], params) assert len(outputs) == 1, f"{caption!r}: expected one output, got {len(outputs)}" audio_mm = outputs[0].multimodal_output assert "audio" in audio_mm, f"{caption!r}: no audio output: {list(audio_mm.keys())}" audio = concat_audio(audio_mm["audio"]) assert audio.size > 0, f"{caption!r}: generated audio is empty" assert np.isfinite(audio).all(), f"{caption!r}: audio contains NaN/Inf" sr_val = audio_mm.get("sr", SAMPLE_RATE) if isinstance(sr_val, list) and sr_val: sr_val = sr_val[-1] if hasattr(sr_val, "item"): sr_val = sr_val.item() assert int(sr_val) == SAMPLE_RATE, f"{caption!r}: unexpected sample_rate={sr_val}" if real_weights: duration_s = audio.size / SAMPLE_RATE # The decode cap is 500 frames = 10 s at 50 fps. assert 0.1 <= duration_s <= 10.5, f"{caption!r}: unexpected duration={duration_s:.3f}s" rms = float(np.sqrt(np.mean(np.square(audio)))) assert rms > 1e-4, f"{caption!r}: audio is near-silent (rms={rms})"