2f69748ee4
Signed-off-by: Bhanu068 <voutharoja.bhanu06@gmail.com> Signed-off-by: Bhanu Prakash Voutharoja <59905694+Bhanu068@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
81 lines
3.0 KiB
Python
81 lines
3.0 KiB
Python
import asyncio
|
|
|
|
from fastapi import Request
|
|
from fastapi.responses import Response
|
|
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
|
from vllm.logger import init_logger
|
|
from vllm.utils import random_uuid
|
|
|
|
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
|
|
from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio, OpenAICreateSpeechRequest
|
|
from vllm_omni.outputs import OmniRequestOutput
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|
async def create_speech(
|
|
self,
|
|
request: OpenAICreateSpeechRequest,
|
|
raw_request: Request | None = None,
|
|
):
|
|
"""
|
|
Create Speech API similar to OpenAI's API.
|
|
|
|
See https://platform.openai.com/docs/api-reference/audio/createSpeech
|
|
for the API specification. This API mimics the OpenAI
|
|
Create Speech API.
|
|
|
|
NOTE: This implementation does not currently support streaming audio
|
|
generation. The full audio is generated before the response is sent.
|
|
"""
|
|
|
|
error_check_ret = await self._check_model(request)
|
|
if error_check_ret is not None:
|
|
logger.error("Error with model %s", error_check_ret)
|
|
return error_check_ret
|
|
|
|
if self.engine_client.errored:
|
|
raise self.engine_client.dead_error
|
|
|
|
request_id = f"speech-{random_uuid()}"
|
|
|
|
try:
|
|
prompt = {"prompt": request.input}
|
|
sampling_params_list = self.engine_client.default_sampling_params_list
|
|
|
|
generator = self.engine_client.generate(
|
|
prompt=prompt, request_id=request_id, sampling_params_list=sampling_params_list
|
|
)
|
|
|
|
final_output: OmniRequestOutput | None = None
|
|
async for res in generator:
|
|
final_output = res
|
|
|
|
if final_output is None:
|
|
return self.create_error_response("No output generated from the model.")
|
|
|
|
if final_output.final_output_type != "audio":
|
|
return self.create_error_response(f"Unexpected final output type: {final_output.final_output_type}")
|
|
|
|
audio_tensor = final_output.request_output.multimodal_output["audio"].float().detach().cpu().numpy()
|
|
audio_obj = CreateAudio(
|
|
audio_tensor=audio_tensor,
|
|
sample_rate=24000,
|
|
response_format=request.response_format,
|
|
speed=request.speed,
|
|
stream_format=request.stream_format,
|
|
base64_encode=False,
|
|
)
|
|
|
|
audio_response: AudioResponse = self.create_audio(audio_obj)
|
|
return Response(content=audio_response.audio_data, media_type=audio_response.media_type)
|
|
|
|
except asyncio.CancelledError:
|
|
return self.create_error_response("Client disconnected")
|
|
except ValueError as e:
|
|
# TODO: Use a vllm-specific Validation Error
|
|
return self.create_error_response(str(e))
|
|
except Exception as e:
|
|
return self.create_error_response(f"{e} {e.__cause__}")
|