Fix "Add online serving to Stable Audio Diffusion and introduce v1/audio/generate endpoint" (#1794)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-04-28 18:06:57 +00:00
1 parent b50fe1d8b1
commit 4c2bea782e
13 files changed
+1788 -7

No files matched your search

+63 -3
View File
@@ -95,7 +95,11 @@ from vllm_omni.entrypoints.openai.image_api_utils import (
parse_size,
validate_layered_layers,
)
from vllm_omni.entrypoints.openai.protocol.audio import BatchSpeechRequest, OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.protocol.audio import (
BatchSpeechRequest,
OpenAICreateAudioGenerateRequest,
OpenAICreateSpeechRequest,
)
from vllm_omni.entrypoints.openai.protocol.images import (
ImageData,
ImageGenerationRequest,
@@ -112,6 +116,7 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
VideoResponse,
)
from vllm_omni.entrypoints.openai.realtime_connection import RealtimeConnection
from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServingAudioGenerate
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
@@ -313,6 +318,9 @@ class _DiffusionServingModels:
def base_model_paths(self) -> list[BaseModelPath]:
return self._base_model_paths
def is_base_model(self, model_name: str) -> bool:
return any(p.name == model_name for p in self._base_model_paths)
def __getattr__(self, name):
return self._Unsupported(name)
@@ -581,10 +589,10 @@ async def omni_init_app_state(
# For omni models
state.stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
model_name = served_model_names[0] if served_model_names else args.model
# Pure Diffusion mode: use simplified initialization logic
if is_pure_diffusion:
model_name = served_model_names[0] if served_model_names else args.model
state.vllm_config = None
state.diffusion_engine = engine_client
state.openai_serving_models = _DiffusionServingModels(base_model_paths)
@@ -597,6 +605,16 @@ async def omni_init_app_state(
model_name=model_name,
)
# audio related
state.openai_serving_speech = None
state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate.for_diffusion(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
model_name=model_name,
)
# video related
diffusion_stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion(
diffusion_engine=engine_client, # type: ignore
@@ -906,7 +924,11 @@ async def omni_init_app_state(
)
state.openai_serving_speech = OmniOpenAIServingSpeech(
engine_client, state.openai_serving_models, request_logger=request_logger
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
)
state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate(
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
)
state.openai_streaming_speech = OmniStreamingSpeechHandler(
@@ -949,6 +971,10 @@ def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None:
return request.app.state.openai_serving_speech
def OmniAudioGenerate(request: Request) -> OmniOpenAIServingAudioGenerate | None:
return getattr(request.app.state, "openai_serving_audio_generate", None)
@router.post(
"/v1/chat/completions",
dependencies=[Depends(validate_json_request)],
@@ -1115,6 +1141,40 @@ async def create_speech_batch(request: BatchSpeechRequest, raw_request: Request)
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
@router.post(
"/v1/audio/generate",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"audio/*": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_audio_generate(request: OpenAICreateAudioGenerateRequest, raw_request: Request):
handler = OmniAudioGenerate(raw_request)
if handler is None:
base_server = getattr(raw_request.app.state, "openai_serving_tokenization", None)
if base_server is None:
raise HTTPException(
status_code=HTTPStatus.NOT_FOUND.value,
detail="The model does not support Audio Generate API",
)
return base_server.create_error_response(message="The model does not support Audio Generate API")
try:
result = await handler.create_audio_generate(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(
content=result.model_dump(),
status_code=result.error.code if result.error else 400,
)
return result
except Exception as e:
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
@router.get(
"/v1/audio/voices",
responses={
@@ -42,6 +42,10 @@ class AudioMixin:
"Only mono (1D) and stereo (2D) are supported."
)
if audio_tensor.ndim == 2 and audio_tensor.shape[0] == 2:
# Convert from [channels, samples] to [samples, channels]
audio_tensor = audio_tensor.T
audio_tensor, sample_rate = self._apply_speed_adjustment(audio_tensor, speed, sample_rate)
supported_formats = {
@@ -124,6 +124,53 @@ class OpenAICreateSpeechRequest(BaseModel):
return self
class OpenAICreateAudioGenerateRequest(BaseModel):
"""Request model for audio generation via diffusion models (e.g. Stable Audio)."""
input: str = Field(
description="Text prompt describing the audio to generate",
)
model: str | None = None
response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
speed: float | None = Field(
default=1.0,
ge=0.25,
le=4.0,
)
stream_format: Literal["sse", "audio"] | None = "audio"
audio_length: float | None = Field(
default=None,
description="Audio length in seconds",
)
audio_start: float | None = Field(
default=0.0,
description="Audio start time in seconds",
)
negative_prompt: str | None = Field(
default=None,
description="Negative prompt for classifier-free guidance",
)
guidance_scale: float | None = Field(
default=None,
description="Guidance scale for diffusion models",
)
num_inference_steps: int | None = Field(
default=None,
description="Number of inference steps",
)
seed: int | None = Field(
default=None,
description="Random seed for reproducibility",
)
@field_validator("stream_format")
@classmethod
def validate_stream_format(cls, v: str) -> str:
if v == "sse":
raise ValueError("'sse' is not a supported stream_format yet. Please use 'audio'.")
return v
class CreateAudio(BaseModel):
audio_tensor: np.ndarray
sample_rate: int = 24000
@@ -0,0 +1,168 @@
import asyncio
import torch
from fastapi import Request
from fastapi.responses import Response
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.logger import init_logger
from vllm.utils import random_uuid
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
from vllm_omni.entrypoints.openai.protocol.audio import (
AudioResponse,
CreateAudio,
OpenAICreateAudioGenerateRequest,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
class OmniOpenAIServingAudioGenerate(OpenAIServing, AudioMixin):
"""Serving class for audio generation via diffusion models (e.g. Stable Audio)."""
def __init__(self, *args, **kwargs):
self.model_name = kwargs.pop("model_name", None)
super().__init__(*args, **kwargs)
self.diffusion_mode = False
@classmethod
def for_diffusion(cls, *args, **kwargs) -> "OmniOpenAIServingAudioGenerate":
"""Create an instance configured to run in diffusion mode."""
instance = cls(*args, **kwargs)
instance.diffusion_mode = True
return instance
def _is_stable_audio_model(self) -> bool:
return self.engine_client.model_type == "StableAudioPipeline"
async def create_audio_generate(
self,
request: OpenAICreateAudioGenerateRequest,
raw_request: Request | None = None,
):
"""
Generate audio using diffusion-based models (e.g. Stable Audio).
This endpoint is designed for audio generation models as
opposed to TTS models that specifically generate speech.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
logger.error("Error with model %s", error_check_ret)
return error_check_ret
if self.engine_client.errored:
raise self.engine_client.dead_error
request_id = f"audiogen-{random_uuid()}"
try:
default_sr = 44100 # Default sample rate for Stable Audio
# Build prompt for diffusion audio generation
prompt = {
"prompt": request.input,
}
if request.negative_prompt:
prompt["negative_prompt"] = request.negative_prompt
# Build sampling params for diffusion
sampling_params_list = [OmniDiffusionSamplingParams(num_outputs_per_prompt=1)]
# Create generator if seed provided
if request.seed is not None:
from vllm_omni.platforms import current_omni_platform
rng = torch.Generator(device=current_omni_platform.device_type).manual_seed(request.seed)
sampling_params_list[0].generator = rng
if request.guidance_scale is not None:
sampling_params_list[0].guidance_scale = request.guidance_scale
if request.num_inference_steps is not None:
sampling_params_list[0].num_inference_steps = request.num_inference_steps
# Set up audio duration parameters
if request.audio_length is not None:
audio_length = request.audio_length
audio_start = request.audio_start if request.audio_start is not None else 0.0
audio_end_in_s = audio_start + audio_length
sampling_params_list[0].extra_args = {
"audio_start_in_s": audio_start,
"audio_end_in_s": audio_end_in_s,
}
logger.info(
"Audio generation request %s: prompt=%r",
request_id,
request.input[:50] + "..." if len(request.input) > 50 else request.input,
)
generator = self.engine_client.generate(
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
output_modalities=["audio"],
)
final_output: OmniRequestOutput | None = None
async for res in generator:
final_output = res
if final_output is None:
return self.create_error_response("No output generated from the model.")
# Extract audio from output
audio_output = None
if hasattr(final_output, "multimodal_output") and final_output.multimodal_output:
audio_output = final_output.multimodal_output
if not audio_output and hasattr(final_output, "request_output"):
if final_output.request_output and hasattr(final_output.request_output, "multimodal_output"):
audio_output = final_output.request_output.multimodal_output
# Check for audio data using either "audio" or "model_outputs" key
audio_key = None
if audio_output:
if "audio" in audio_output:
audio_key = "audio"
elif "model_outputs" in audio_output:
audio_key = "model_outputs"
if not audio_output or audio_key is None:
return self.create_error_response("Audio generation model did not produce audio output.")
audio_tensor = audio_output[audio_key]
sample_rate = audio_output.get("sr", default_sr)
if hasattr(sample_rate, "item"):
sample_rate = sample_rate.item()
# Convert tensor to numpy
if hasattr(audio_tensor, "float"):
audio_tensor = audio_tensor.float().detach().cpu().numpy()
# Squeeze batch dimension if present, but preserve channel dimension for stereo
if audio_tensor.ndim > 1:
audio_tensor = audio_tensor.squeeze()
audio_obj = CreateAudio(
audio_tensor=audio_tensor,
sample_rate=int(sample_rate),
response_format=request.response_format or "wav",
speed=request.speed or 1.0,
stream_format=request.stream_format,
base64_encode=False,
)
audio_response: AudioResponse = self.create_audio(audio_obj)
return Response(content=audio_response.audio_data, media_type=audio_response.media_type)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
except Exception as e:
logger.exception("Audio generation failed: %s", e)
return self.create_error_response(f"Audio generation failed: {e}")
@@ -197,6 +197,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return instance
def __init__(self, *args, **kwargs):
self.model_name = kwargs.pop("model_name", None)
super().__init__(*args, **kwargs)
# Initialize uploaded speakers storage (ephemeral — cleared on restart)
speech_voice_samples_dir = os.environ.get("SPEECH_VOICE_SAMPLES", "/tmp/voice_samples")