From 72beeaee869f19bede9f10077414e1d950845599 Mon Sep 17 00:00:00 2001 From: Jae Date: Tue, 25 Aug 2026 10:02:26 -0400 Subject: [PATCH] [Misc] move text-to-audio online examples to a unified folder (#4807) Signed-off-by: zzehli --- .../examples/online_serving/text_to_audio.md | 193 --------------- .../online_serving/stable_audio/README.md | 234 ------------------ .../stable_audio/curl_examples.sh | 54 ---- .../stable_audio/stable_audio_client.py | 170 ------------- recipes/StabilityAI/Stable-Audio-Open.md | 8 +- .../test_stable_audio_online_expansion.py | 71 ++++++ 6 files changed, 74 insertions(+), 656 deletions(-) delete mode 100644 docs/user_guide/examples/online_serving/text_to_audio.md delete mode 100644 examples/online_serving/stable_audio/README.md delete mode 100755 examples/online_serving/stable_audio/curl_examples.sh delete mode 100755 examples/online_serving/stable_audio/stable_audio_client.py create mode 100644 tests/e2e/online_serving/test_stable_audio_online_expansion.py diff --git a/docs/user_guide/examples/online_serving/text_to_audio.md b/docs/user_guide/examples/online_serving/text_to_audio.md deleted file mode 100644 index a4833aef6..000000000 --- a/docs/user_guide/examples/online_serving/text_to_audio.md +++ /dev/null @@ -1,193 +0,0 @@ -# Text-To-Audio - -Source . - -This example demonstrates how to deploy Stable Audio models for online text-to-audio generation using vLLM-Omni. - -## Supported Models - -| Model | Description | -|-------|-------------| -| `stabilityai/stable-audio-open-1.0` | Open-source audio generation, up to ~47 seconds, 44.1 kHz stereo | - -## Start Server - -### Basic Start - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8091 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni -``` - -## API Calls - -### Method 1: Using curl - -```bash -# Run all curl examples -bash curl_examples.sh - -# Or execute directly -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a cat purring", - "audio_length": 10.0 - }' --output cat.wav -``` - -### Method 2: Using Python Client - -```bash -cd examples/online_serving/stable_audio - -# Simple generation -python stable_audio_client.py \ - --text "The sound of a cat purring" - -# With custom duration -python stable_audio_client.py \ - --text "A dog barking" \ - --audio_length 5.0 - -# With all parameters -python stable_audio_client.py \ - --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -``` - -The Python client supports the following command-line arguments: - -- `--api_url`: API endpoint URL (default: `http://localhost:8091/v1/audio/generate`) -- `--text`: Text prompt for audio generation (default: `"The sound of a cat purring"`) -- `--audio_length`: Audio length in seconds (default: `10.0`, max ~47s for `stable-audio-open-1.0`) -- `--audio_start`: Audio start time in seconds (default: `0.0`) -- `--negative_prompt`: Negative prompt for classifier-free guidance (default: `"Low quality"`) -- `--guidance_scale`: Guidance scale for diffusion (default: `7.0`) -- `--num_inference_steps`: Number of inference steps (default: `100`) -- `--seed`: Random seed for reproducibility (default: `None`) -- `--response_format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm` -- `--output`: Output file path (default: `stable_audio_output.wav`) - -### Method 3: Using Python httpx - -```python -import httpx - -response = httpx.post( - "http://localhost:8091/v1/audio/generate", - json={ - "input": "The sound of ocean waves crashing on a beach", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted", - "guidance_scale": 7.0, - "num_inference_steps": 100, - }, - timeout=300.0, -) - -with open("ocean.wav", "wb") as f: - f.write(response.content) -``` - -## Request Format - -### Simple Generation - -```json -{ - "input": "The sound of ocean waves" -} -``` - -### Generation with Parameters - -```json -{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy", - "guidance_scale": 8.0, - "num_inference_steps": 150, - "seed": 42, - "response_format": "wav" -} -``` - -## API Reference - -### Endpoint - -``` -POST /v1/audio/generate -Content-Type: application/json -``` - -### Generation Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `input` | string | **required** | Text prompt describing the audio to generate | -| `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, opus | -| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | -| `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) | -| `audio_start` | float | 0.0 | Audio start time in seconds | -| `negative_prompt` | string | null | Text describing what to avoid in generation | -| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) | -| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) | -| `seed` | int | null | Random seed for reproducible generation | - -### Response Format - -Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`). - -## Tuning Tips - -1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`. -2. **Quality vs Speed**: - - 50 steps: Fast, decent quality (quick previews) - - 100 steps: Good balance (general purpose) - - 150+ steps: High quality, slower (final / critical audio) -3. **Guidance Scale**: - - Lower (3 - 5): More creative / varied output - - Default (7): Good balance - - Higher (10+): Strict adherence to the prompt -4. **Negative Prompts**: Use to avoid unwanted characteristics such as `"Low quality"`, `"distorted"`, `"noisy"`. -5. **Seeds**: Set a fixed seed to get deterministic, reproducible results. - -## File Description - -| File | Description | -|------|-------------| -| `curl_examples.sh` | Curl examples covering common use cases | -| `stable_audio_client.py` | Python client with full CLI argument support | - -## Troubleshooting - -1. **Audio generation model did not produce audio output**: Verify the server started successfully and the model loaded without errors. -2. **Connection refused**: Make sure the server is running on the correct port. -3. **Generation timeout**: Reduce `num_inference_steps` or `audio_length`, and check GPU memory with `nvidia-smi`. -4. **Out of memory**: Lower `--gpu-memory-utilization` or reduce `audio_length`. -5. **Audio quality issues**: Increase `num_inference_steps`, add a negative prompt, or raise `guidance_scale`. - -## Example materials - -??? abstract "stable_audio_client.py" - ``````py - --8<-- "examples/online_serving/stable_audio/stable_audio_client.py" - `````` -??? abstract "curl_examples.sh" - ``````sh - --8<-- "examples/online_serving/stable_audio/curl_examples.sh" - `````` diff --git a/examples/online_serving/stable_audio/README.md b/examples/online_serving/stable_audio/README.md deleted file mode 100644 index 3aaf34e56..000000000 --- a/examples/online_serving/stable_audio/README.md +++ /dev/null @@ -1,234 +0,0 @@ -# Stable Audio Online Serving - -Generate audio from text prompts using Stable Audio models via an OpenAI-compatible API endpoint. - -## Features - -- **OpenAI-compatible API**: Use `/v1/audio/generate` endpoint -- **Flexible control**: Adjust audio length, guidance scale, inference steps -- **Quality control**: Use negative prompts to avoid unwanted characteristics -- **Reproducible**: Set random seed for deterministic generation - -## Quick Start - -### 1. Start the Server - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8091 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni -``` - -### 2. Generate Audio - -#### Using curl - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a cat purring", - "audio_length": 10.0 - }' --output cat.wav -``` - -#### Using Python Client - -```bash -python stable_audio_client.py \ - --text "The sound of a cat purring" \ - --audio_length 10.0 \ - --output cat.wav -``` - -#### Using Bash Script - -```bash -bash curl_examples.sh -``` - -## API Reference - -### Endpoint - -``` -POST /v1/audio/generate -``` - -### Request Body - -```json -{ - "input": "Text description of the audio", - "audio_length": 10.0, - "audio_start": 0.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42, - "response_format": "wav" -} -``` - -### Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `input` | string | **required** | Text prompt describing the audio to generate | -| `audio_length` | float | ~47s | Audio duration in seconds (max ~47s for stable-audio-open-1.0) | -| `audio_start` | float | 0.0 | Audio start time in seconds | -| `negative_prompt` | string | null | Text describing what to avoid in generation | -| `guidance_scale` | float | 7.0 | Classifier-free guidance scale (higher = more adherence to prompt) | -| `num_inference_steps` | int | 50 | Number of denoising steps (higher = better quality, slower) | -| `seed` | int | null | Random seed for reproducibility | -| `response_format` | string | "wav" | Output format: wav, mp3, flac, pcm | - -### Response - -Returns audio data in the requested format (default: WAV). - -## Usage Examples - -### Basic Generation - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of ocean waves" - }' --output ocean.wav -``` - -### Custom Duration - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A dog barking", - "audio_length": 5.0 - }' --output dog_5s.wav -``` - -### High Quality with Negative Prompt - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy", - "guidance_scale": 8.0, - "num_inference_steps": 150 - }' --output piano_hq.wav -``` - -### Reproducible Generation - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "seed": 42 - }' --output thunder.wav -``` - -### Quick Generation (Fewer Steps) - -For faster generation with slightly lower quality: - -```bash -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Birds chirping in a forest", - "audio_length": 8.0, - "num_inference_steps": 50 - }' --output birds_quick.wav -``` - -## Python Client Examples - -### Simple Generation - -```bash -python stable_audio_client.py \ - --text "The sound of a cat purring" -``` - -### Custom Parameters - -```bash -python stable_audio_client.py \ - --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -``` - -### Different Output Format - -```bash -python stable_audio_client.py \ - --text "Guitar playing" \ - --response_format mp3 \ - --output guitar.mp3 -``` - -## Tips - -1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0` -2. **Quality vs Speed**: - - 50 steps: Fast, decent quality - - 100 steps: Good balance (default) - - 150+ steps: High quality, slower -3. **Guidance Scale**: - - Lower (3-5): More creative/varied - - Default (7): Good balance - - Higher (10+): More literal to prompt -4. **Negative Prompts**: Use to avoid "Low quality", "distorted", "noisy", etc. -5. **Seeds**: Use same seed for reproducible results - -## Performance - -| Inference Steps | Quality | Speed | Use Case | -|----------------|---------|-------|----------| -| 50 | Good | Fast | Quick previews | -| 100 (default) | Very Good | Medium | Production | -| 150+ | Excellent | Slow | Final/critical audio | - -## Troubleshooting - -### Server not responding -- Check if server is running: `curl http://localhost:8091/health` -- Check server logs for errors - -### Audio quality issues -- Increase `num_inference_steps` (e.g., 150) -- Add negative prompts: `"Low quality, distorted, noisy"` -- Increase `guidance_scale` for more prompt adherence - -### Generation timeout -- Reduce `num_inference_steps` -- Reduce `audio_length` -- Check GPU memory with `nvidia-smi` - -### Wrong audio length -- Ensure `audio_length` is within model limits (~47s max) -- Adjust `audio_start` if trimming is needed - -## See Also - -- [Offline Inference Example](../../offline_inference/text_to_audio/README.md) -- [Stable Audio Model Card](https://huggingface.co/stabilityai/stable-audio-open-1.0) -- [vLLM-Omni Documentation](https://github.com/vllm-project/vllm-omni) diff --git a/examples/online_serving/stable_audio/curl_examples.sh b/examples/online_serving/stable_audio/curl_examples.sh deleted file mode 100755 index 16a12ef1c..000000000 --- a/examples/online_serving/stable_audio/curl_examples.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Examples for using Stable Audio with curl via /v1/audio/generate endpoint - -# Example 1: Simple request with default parameters -echo "Example 1: Simple request with default parameters" -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound audience clapping and cheering in a stadium" - }' --output stadium.wav - -# Example 2: Request with custom audio_length -echo "Example 2: Custom audio length (5 seconds)" -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a dog barking", - "audio_length": 5.0 - }' --output dog_5s.wav - -# Example 3: Request with negative prompt for quality control -echo "Example 3: With negative prompt" -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy" - }' --output piano.wav - -# Example 4: Full control with all parameters -echo "Example 4: Full control (custom length, guidance, steps, seed)" -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42 - }' --output thunder_rain.wav - -# Example 5: Quick generation with fewer steps (faster but lower quality) -echo "Example 5: Quick generation (fewer steps)" -curl -X POST http://localhost:8091/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Ocean waves crashing on a beach", - "audio_length": 8.0, - "num_inference_steps": 50 - }' --output ocean.wav - -echo "All examples completed!" diff --git a/examples/online_serving/stable_audio/stable_audio_client.py b/examples/online_serving/stable_audio/stable_audio_client.py deleted file mode 100755 index 7f28a5245..000000000 --- a/examples/online_serving/stable_audio/stable_audio_client.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -""" -OpenAI-compatible client for Stable Audio via /v1/audio/generate endpoint. - -This script demonstrates how to use the OpenAI-compatible speech API -to generate audio from text using Stable Audio models. - -Examples: - # Simple generation - python stable_audio_client.py --text "The sound of a cat purring" - - # With custom duration - python stable_audio_client.py --text "A dog barking" --audio_length 5.0 - - # With all parameters - python stable_audio_client.py --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -""" - -import argparse -import sys - -import requests - - -def parse_args(): - parser = argparse.ArgumentParser(description="Generate audio with Stable Audio via OpenAI-compatible API") - parser.add_argument( - "--api_url", - default="http://localhost:8091/v1/audio/generate", - help="API endpoint URL", - ) - parser.add_argument( - "--text", - default="The sound of a cat purring", - help="Text prompt for audio generation", - ) - parser.add_argument( - "--audio_length", - type=float, - default=10.0, - help="Audio length in seconds (max ~47s for stable-audio-open-1.0)", - ) - parser.add_argument( - "--audio_start", - type=float, - default=0.0, - help="Audio start time in seconds", - ) - parser.add_argument( - "--negative_prompt", - default="Low quality", - help="Negative prompt for classifier-free guidance", - ) - parser.add_argument( - "--guidance_scale", - type=float, - default=7.0, - help="Guidance scale for diffusion (higher = more adherence to prompt)", - ) - parser.add_argument( - "--num_inference_steps", - type=int, - default=100, - help="Number of inference steps (higher = better quality, slower)", - ) - parser.add_argument( - "--seed", - type=int, - default=None, - help="Random seed for reproducibility", - ) - parser.add_argument( - "--output", - default="stable_audio_output.wav", - help="Output file path", - ) - parser.add_argument( - "--response_format", - default="wav", - choices=["wav", "mp3", "flac", "pcm"], - help="Audio output format", - ) - return parser.parse_args() - - -def generate_audio(args): - """Generate audio using the API.""" - - # Build request payload - payload = { - "input": args.text, - "audio_length": args.audio_length, - "audio_start": args.audio_start, - "response_format": args.response_format, - } - - # Add optional parameters - if args.negative_prompt: - payload["negative_prompt"] = args.negative_prompt - if args.guidance_scale: - payload["guidance_scale"] = args.guidance_scale - if args.num_inference_steps: - payload["num_inference_steps"] = args.num_inference_steps - if args.seed is not None: - payload["seed"] = args.seed - - print(f"\n{'=' * 60}") - print("Stable Audio - Text-to-Audio Generation") - print(f"{'=' * 60}") - print(f"API URL: {args.api_url}") - print(f"Prompt: {args.text}") - print(f"Audio length: {args.audio_length}s") - print(f"Negative prompt: {args.negative_prompt}") - print(f"Guidance scale: {args.guidance_scale}") - print(f"Inference steps: {args.num_inference_steps}") - if args.seed is not None: - print(f"Seed: {args.seed}") - print(f"Output: {args.output}") - print(f"{'=' * 60}\n") - - try: - # Make the API request - print("Generating audio...") - response = requests.post( - args.api_url, - json=payload, - headers={"Content-Type": "application/json"}, - timeout=300, # 5 minute timeout for long generations - ) - - # Check for errors - if response.status_code != 200: - print(f"Error: API returned status code {response.status_code}") - print(f"Response: {response.text}") - return False - - # Save the audio - with open(args.output, "wb") as f: - f.write(response.content) - - print(f"✓ Audio saved to {args.output}") - print(f" File size: {len(response.content) / 1024:.1f} KB") - return True - - except requests.exceptions.Timeout: - print("Error: Request timed out. Try reducing inference steps or audio length.") - return False - except requests.exceptions.ConnectionError: - print(f"Error: Could not connect to {args.api_url}") - print("Make sure the server is running.") - return False - except Exception as e: - print(f"Error: {e}") - return False - - -def main(): - args = parse_args() - success = generate_audio(args) - sys.exit(0 if success else 1) - - -if __name__ == "__main__": - main() diff --git a/recipes/StabilityAI/Stable-Audio-Open.md b/recipes/StabilityAI/Stable-Audio-Open.md index 4b5fd59c2..d24d516ca 100644 --- a/recipes/StabilityAI/Stable-Audio-Open.md +++ b/recipes/StabilityAI/Stable-Audio-Open.md @@ -23,8 +23,6 @@ offline validation sample with TeaCache and online serving through the - Model: - Offline example: [`examples/offline_inference/text_to_audio`](../../examples/offline_inference/text_to_audio) -- Online example: - [`examples/online_serving/stable_audio`](../../examples/online_serving/stable_audio) - Audio generation API: [`docs/serving/audio_generate_api.md`](../../docs/serving/audio_generate_api.md) - Related issue: @@ -103,7 +101,7 @@ curl -X POST http://localhost:8091/v1/audio/generate \ "seed": 42, "response_format": "wav" }' \ - --output examples/online_serving/stable_audio/piano_10s.wav + --output piano_10s.wav ``` ##### Verification @@ -137,12 +135,12 @@ PY Validate the online output: ```bash -ls -lh examples/online_serving/stable_audio/piano_10s.wav +ls -lh piano_10s.wav python - <<'PY' import soundfile as sf -path = "examples/online_serving/stable_audio/piano_10s.wav" +path = "piano_10s.wav" audio, sample_rate = sf.read(path) print("sample_rate:", sample_rate) print("shape:", audio.shape) diff --git a/tests/e2e/online_serving/test_stable_audio_online_expansion.py b/tests/e2e/online_serving/test_stable_audio_online_expansion.py new file mode 100644 index 000000000..fbd6bdbfd --- /dev/null +++ b/tests/e2e/online_serving/test_stable_audio_online_expansion.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E online serving test for Stable Audio Open text-to-audio diffusion. + +Stable Audio Open is served through the OpenAI-compatible +`POST /v1/audio/generate` endpoint (JSON in, binary WAV out). +""" + +import os + +import pytest + +from tests.helpers import skip_if_gated_repo_inaccessible +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler + +# Stable Audio Open is gated; override to a local bundle locally if needed. +STABLE_AUDIO_TEST_MODEL = os.environ.get("STABLE_AUDIO_TEST_MODEL", "stabilityai/stable-audio-open-1.0") +T2A_PROMPT = "A piano playing a gentle melody with soft room ambience." + +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}) + + +def _stable_audio_server_cases(model: str): + return [ + pytest.param( + OmniServerParams(model=model), + id="t2a", + marks=SINGLE_CARD_FEATURE_MARKS, + ), + ] + + +@pytest.fixture +def _require_stable_audio_access() -> None: + """Skip cleanly (before the server boots) if the gated checkpoint is inaccessible.""" + skip_if_gated_repo_inaccessible(STABLE_AUDIO_TEST_MODEL) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _stable_audio_server_cases(STABLE_AUDIO_TEST_MODEL), indirect=True) +def test_stable_audio_t2a_online( + _require_stable_audio_access: None, + omni_server: OmniServer, + openai_client: OpenAIClientHandler, +) -> None: + """Stable Audio Open text-to-audio: `/v1/audio/generate` returns a non-empty WAV. + + Uses tiny steps / short duration to keep CI light, matching the offline smoke + test in `tests/e2e/offline_inference/test_stable_audio_expansion.py`. + """ + request_config = { + "json": { + "model": omni_server.model, + "input": T2A_PROMPT, + "audio_length": 2.0, + "num_inference_steps": 4, + "guidance_scale": 7.0, + "negative_prompt": "Low quality.", + "seed": 42, + "response_format": "wav", + }, + "timeout": 300, + } + responses = openai_client.send_audio_generate_http_request(request_config) + assert responses, "no response from /v1/audio/generate" + resp = responses[0] + assert resp.success, f"audio generate failed: {resp.status_code} {resp.error_message}" + assert resp.status_code == 200