""" - Make sure to install the following for this example to function correctly: - `pip install -e .` - `pip install gradio==5.50 mistral_common=1.10.0` Example use case: python examples/online_serving/voxtral_tts/gradio_demo.py --host slurm-199-077 --port 8000 """ import argparse import io import json import logging import socket import time import uuid from datetime import datetime from pathlib import Path from typing import Any try: import gradio as gr except ImportError: raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None import httpx import numpy as np import soundfile as sf from text_preprocess import sanitize_tts_input_text_for_demo logger = logging.getLogger() LOGFORMAT = "%(asctime)s - %(levelname)s - %(message)s" TIMEFORMAT = "%Y-%m-%d %H:%M:%S" logging.basicConfig(level=logging.INFO, force=True, format=LOGFORMAT, datefmt=TIMEFORMAT) logger.setLevel(logging.INFO) _BASE_URL = f"{socket.gethostname()}:7860" # Default fallback voices - comprehensive list _DEFAULT_VOICES = [ "casual_female", "casual_male", "cheerful_female", "neutral_female", "neutral_male", "ar_male", "de_female", "de_male", "es_female", "es_male", "fr_female", "fr_male", "hi_female", "hi_male", "it_female", "it_male", "nl_female", "nl_male", "pt_female", "pt_male", ] def organize_voices_by_language(voices: list[str]) -> tuple[list[str], dict[str, list[str]]]: """Organize voices into language categories. Args: voices: List of voice names (e.g., ["neutral_male", "es_female", "fr_male"]) Returns: Tuple of (sorted list of language categories, dictionary mapping language to voices) """ # Define language prefixes and their display names LANGUAGE_PREFIXES = { "ar": "Arabic", "de": "German", "es": "Spanish", "fr": "French", "it": "Italian", "nl": "Dutch", "pt": "Portuguese", "hi": "Hindi", } # Initialize language to voices mapping language_voices: dict[str, list[str]] = {} for voice in voices: # Check for language prefix found_language = None for prefix, lang_name in LANGUAGE_PREFIXES.items(): if voice.lower().startswith(f"{prefix}_"): found_language = lang_name break if found_language: # Add voice to its language category if found_language not in language_voices: language_voices[found_language] = [] language_voices[found_language].append(voice) else: # Add to English (voices without language prefix) if "English" not in language_voices: language_voices["English"] = [] language_voices["English"].append(voice) # Sort voices within each language category, with neutral_male first for English for lang in language_voices: if lang == "English": language_voices[lang].sort(key=lambda v: (0 if v == "neutral_male" else 1, v)) else: language_voices[lang].sort() # Sort language categories (English first, then alphabetically) sorted_languages = sorted(language_voices.keys(), key=lambda x: (0 if x == "English" else 1, x.lower())) return sorted_languages, language_voices # Configuration for server health check _SERVER_CHECK_TIMEOUT = 300.0 # 5 minutes max wait _SERVER_CHECK_INTERVAL = 5.0 # 5 seconds between retries def wait_for_server(base_url: str, timeout: float = _SERVER_CHECK_TIMEOUT) -> bool: """Block until the server is available or timeout is reached. Args: base_url: Base URL of the server (e.g., "http://localhost:8091/v1") timeout: Maximum time to wait in seconds Returns: True if server became available, False if timeout reached """ start_time = time.time() # Health endpoint is at /health (not /v1/health) health_url = base_url.replace("/v1", "") + "/health" logger.info(f"Waiting for server at {base_url} to become available...") with httpx.Client(timeout=5.0) as client: while time.time() - start_time < timeout: try: resp = client.get(health_url) if resp.status_code == 200: logger.info("Server is now available!") return True except Exception: pass elapsed = time.time() - start_time logger.info(f"Server not yet available ({elapsed:.1f}s elapsed), retrying in {_SERVER_CHECK_INTERVAL}s...") time.sleep(_SERVER_CHECK_INTERVAL) logger.warning(f"Server did not become available within {timeout}s timeout") return False def fetch_voices_and_languages(base_url: str, model: str) -> tuple[list[str], dict[str, list[str]]]: """Fetch available voices from the server API and organize by language. This function blocks until the server is available, then fetches the list of available voices from the /v1/audio/voices endpoint and organizes them. Args: base_url: Base URL of the server API (e.g., "http://localhost:8091/v1") model: Model name (used for logging) Returns: Tuple of (sorted language list, dictionary mapping language to voices) """ # Always wait for server - if server is not up, demo cannot do anything if not wait_for_server(base_url): logger.warning("Server unavailable, using fallback voices") languages, language_voices = organize_voices_by_language(_DEFAULT_VOICES) return languages, language_voices try: with httpx.Client(timeout=10.0) as client: # The audio/voices endpoint is directly under /v1, not nested resp = client.get( f"{base_url}/audio/voices", headers={"Authorization": "Bearer EMPTY"}, ) if resp.status_code == 200: data = resp.json() voices = data.get("voices", []) if voices: logger.info(f"Fetched {len(voices)} voices from server: {voices}") languages, language_voices = organize_voices_by_language(sorted(voices)) return languages, language_voices logger.warning("Server returned empty voices list") except Exception as e: logger.warning(f"Failed to fetch voices from server: {e}") # Fallback to default voices logger.info(f"Using fallback voices: {_DEFAULT_VOICES}") languages, language_voices = organize_voices_by_language(_DEFAULT_VOICES) return languages, language_voices def make_update_voice_dropdown(language_voices: dict[str, list[str]]): """Return a callback that updates the voice dropdown when the user selects a different language.""" def update_voice_dropdown(language: str) -> gr.Dropdown: voices = language_voices.get(language, []) return gr.Dropdown(choices=voices, value=voices[0] if voices else None, interactive=True) return update_voice_dropdown def run_inference( voice_name: str, text_prompt: str, cfg_alpha: float, base_url: str, model: str, ) -> tuple[int, np.ndarray]: """Call /v1/audio/speech and return (sample_rate, audio_array).""" user_text_prompt = text_prompt.strip() if not user_text_prompt: raise gr.Error("Please enter a text prompt.") try: text_prompt = sanitize_tts_input_text_for_demo(user_text_prompt) except Exception as exc: raise gr.Error(f"Text preprocessing failed: {exc}") from exc payload: dict[str, Any] = { "input": text_prompt, "model": model, "response_format": "wav", "voice": voice_name, "extra_params": {"cfg_alpha": cfg_alpha}, } response = httpx.post( f"{base_url}/audio/speech", json=payload, timeout=120.0, ) response.raise_for_status() audio_array, sr = sf.read(io.BytesIO(response.content), dtype="float32") return sr, audio_array def _save_example( outputs_dir: Path, voice_name: str, text_prompt: str, sr: int, audio_array: np.ndarray, ) -> tuple[str, str]: """ Save inputs/outputs for sharing. Returns (share_id, saved_audio_path) """ share_id = uuid.uuid4().hex # Save generated audio saved_audio_path = outputs_dir / f"{share_id}_gen.wav" sf.write(str(saved_audio_path), audio_array, sr) meta = { "id": share_id, "created_at": datetime.utcnow().isoformat(), "voice_name": voice_name, "text_prompt": text_prompt, "generated_audio_path": str(saved_audio_path), } with open(outputs_dir / f"{share_id}.json", "w") as f: json.dump(meta, f, ensure_ascii=False) return share_id, str(saved_audio_path) def _load_from_share( outputs_dir: Path | None, request: gr.Request, languages: list[str] | None = None, language_voices: dict[str, list[str]] | None = None, ) -> tuple[str, str, str | None, str | None, dict[str, Any], str]: """ Called on page load. If ?share_id=... is present, load stored example. Returns: (language, voice_name, text_prompt, output_audio, submit_btn_update, share_link_text) """ fallback_language = ( "English" if languages and "English" in languages else (languages[0] if languages else "English") ) if language_voices: voices_list = language_voices.get(fallback_language, []) fallback_voice = voices_list[0] if voices_list else None else: fallback_voice = None if outputs_dir is None: return fallback_language, fallback_voice, "", None, gr.update(interactive=False), "" share_id = None if request and request.query_params: share_id = request.query_params.get("share_id") if not share_id: return fallback_language, fallback_voice, "", None, gr.update(interactive=False), "" meta_path = outputs_dir / f"{share_id}.json" if not meta_path.exists(): logger.warning("No stored example for share_id=%s", share_id) return fallback_language, fallback_voice, "", None, gr.update(interactive=False), "" with open(meta_path) as f: meta = json.load(f) # Determine language and voice from the stored voice name voice = meta.get("voice_name", fallback_voice) text_prompt = meta.get("text_prompt", "") gen_path = meta.get("generated_audio_path") # Find which language this voice belongs to language = fallback_language if language_voices: for lang, voices in language_voices.items(): if voice in voices: language = lang break if _BASE_URL: share_link = f"http://{_BASE_URL}?share_id={share_id}" else: share_link = f"Use this query on your current URL: ?share_id={share_id}" return language, voice, text_prompt, gen_path, gr.update(interactive=True), share_link def main( model: str, host: str, port: str, output_dir: str | None = None, ) -> None: base_url = f"http://{host}:{port}/v1" logger.info(f"Using speech API at: {base_url}/audio/speech") outputs_dir: Path | None = None if output_dir is not None: outputs_dir = Path(output_dir) outputs_dir.mkdir(parents=True, exist_ok=True) # Load available voices and organize by language # This will block until the server is available languages, language_voices = fetch_voices_and_languages(base_url, model) with gr.Blocks(title="Voxtral TTS", fill_height=True) as demo: gr.Markdown("## Voxtral TTS") with gr.Row(): with gr.Column(): # Language dropdown (first level) default_language = "English" if "English" in languages else languages[0] language_dropdown = gr.Dropdown( choices=languages, label="Language", value=default_language, ) # Voice dropdown (second level, updates based on language) voices_for_default_lang = language_voices.get(default_language, []) voice_name = gr.Dropdown( choices=voices_for_default_lang, label="Voice", value=voices_for_default_lang[0] if voices_for_default_lang else None, ) text_prompt = gr.Textbox( label="Text prompt", placeholder="Enter the text you want to synthesize...", lines=4, ) cfg_alpha_slider = gr.Slider( minimum=1.0, maximum=2.0, step=0.1, value=1.2, label="CFG Alpha", info="Flow-matching guidance strength (default: 1.2)", ) with gr.Row(): reset_btn = gr.Button("Clear") submit_btn = gr.Button("Generate audio", interactive=False) with gr.Column(): output_audio = gr.Audio( label="Generated audio", show_download_button=True, interactive=False, autoplay=True, type="filepath", ) share_link_box = gr.Textbox( label="Shareable link", interactive=False, show_copy_button=True, visible=outputs_dir is not None, ) # --- Cascading dropdown: update voice list when language changes --- language_dropdown.change( fn=make_update_voice_dropdown(language_voices), inputs=[language_dropdown], outputs=voice_name, ) # --- UI logic: disable submit until text is non-empty --- def _toggle_submit(text: str): enabled = bool(text.strip()) return gr.update(interactive=enabled) text_prompt.change( fn=_toggle_submit, inputs=[text_prompt], outputs=submit_btn, ) # --- Wiring inference + persistence to the button --- def _on_submit(voice: str, text: str, cfg_alpha: float): assert text.strip() != "" sr, audio_array = run_inference(voice, text, cfg_alpha, base_url, model) if outputs_dir is not None: share_id, saved_audio_path = _save_example( outputs_dir, voice_name=voice, text_prompt=text, sr=sr, audio_array=audio_array, ) share_link = f"{_BASE_URL}?share_id={share_id}" return saved_audio_path, share_link return (sr, audio_array), "" submit_btn.click( fn=_on_submit, inputs=[voice_name, text_prompt, cfg_alpha_slider], outputs=[output_audio, share_link_box], ) # --- Clear everything and disable submit again --- def make_on_reset(languages: list[str], language_voices: dict[str, list[str]]): def _on_reset(): language = "English" if "English" in languages else languages[0] voices_list = language_voices.get(language, []) voice = voices_list[0] if voices_list else None return ( language, # language_dropdown voice, # voice_name "", # text_prompt 1.2, # cfg_alpha_slider None, # output_audio gr.update(interactive=False), # submit_btn "", # share_link_box ) return _on_reset reset_btn.click( fn=make_on_reset(languages, language_voices), inputs=[], outputs=[ language_dropdown, voice_name, text_prompt, cfg_alpha_slider, output_audio, submit_btn, share_link_box, ], ) def make_load_from_share(outputs_dir: Path | None, languages: list[str], language_voices: dict[str, list[str]]): def _load(request: gr.Request): return _load_from_share(outputs_dir, request, languages, language_voices) return _load demo.load( fn=make_load_from_share(outputs_dir, languages, language_voices), inputs=[], outputs=[language_dropdown, voice_name, text_prompt, output_audio, submit_btn, share_link_box], ) launch_kwargs: dict[str, Any] = { "server_name": "0.0.0.0", "share": True, } if outputs_dir is not None: launch_kwargs["allowed_paths"] = [str(outputs_dir)] demo.launch(**launch_kwargs) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Voxtral TTS Gradio Demo") parser.add_argument("--model", type=str, default="mistralai/Voxtral-4B-TTS-2603", help="Name of model repo on HF") parser.add_argument("--host", type=str, default="localhost", help="Name of host") parser.add_argument("--port", type=str, default="8091", help="port number") parser.add_argument( "--output-dir", type=str, default=None, help="Directory to save generated audio and share links. " "If not provided, save/share functionality is disabled.", ) args = parser.parse_args() main( model=args.model, host=args.host, port=args.port, output_dir=args.output_dir, )