[Diffusion] [Model] Support AudioX (#2077)
Signed-off-by: Zhang Jian <jianmusings@gmail.com> Signed-off-by: Zhang <jianmusings@gmail.com> Signed-off-by: Zhang <zhang.jian@u.nus.edu> Signed-off-by: Zhang Jian <e0322744@u.nus.edu> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Zhang Jian <e0322744@u.nus.edu> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
25 files changed
+2487
-66
No files matched your search
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
# Prefer a tiny/random checkpoint for CI.
|
||||
# Override in CI if needed: AUDIOX_TEST_MODEL=<model-or-local-path>
|
||||
models = [os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_audiox_model(model_name: str):
|
||||
m = Omni(model=model_name, model_class_name="AudioXPipeline")
|
||||
|
||||
# Keep runtime short for CI.
|
||||
seconds_total = 2.0
|
||||
# AudioXPipeline always emits 44.1 kHz stereo (advertised via class-level
|
||||
# ``audio_sample_rate``); the trimmed output should match this rate.
|
||||
sample_rate = 44100
|
||||
|
||||
outputs = m.generate(
|
||||
prompts={"prompt": "A dog barking in a quiet park."},
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
num_inference_steps=4,
|
||||
guidance_scale=6.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
extra_args={
|
||||
"audiox_task": "t2a",
|
||||
"seconds_start": 0.0,
|
||||
"seconds_total": seconds_total,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
assert outputs is not None
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "audio"
|
||||
assert hasattr(first_output, "request_output") and first_output.request_output
|
||||
|
||||
req_out = first_output.request_output
|
||||
assert isinstance(req_out, OmniRequestOutput)
|
||||
assert req_out.final_output_type == "audio"
|
||||
assert hasattr(req_out, "multimodal_output") and req_out.multimodal_output
|
||||
|
||||
audio = req_out.multimodal_output.get("audio")
|
||||
assert isinstance(audio, np.ndarray)
|
||||
# audio shape: (batch, channels, samples)
|
||||
assert audio.ndim == 3
|
||||
assert audio.shape[0] == 1
|
||||
assert audio.shape[1] == 2
|
||||
assert audio.shape[2] > 0
|
||||
expected_samples = int(seconds_total * sample_rate)
|
||||
assert abs(audio.shape[2] - expected_samples) <= 2 * 1024
|
||||
@@ -56,13 +56,9 @@ def generate_stable_audio_short_clip(
|
||||
|
||||
assert outputs is not None
|
||||
first_output = outputs[0]
|
||||
# Outer OmniRequestOutput.final_output_type comes from get_stage_metadata.
|
||||
# The nested request_output is the worker OmniRequestOutput
|
||||
# (e.g. final_output_type="audio") and holds the multimodal payload.
|
||||
# Follow-up: add StableAudioPipeline stage YAML, and pass model into
|
||||
# _create_default_diffusion_stage_cfg so default diffusion metadata can set
|
||||
# final_output_type to "audio" for future audio pipelines without YAML.
|
||||
assert first_output.final_output_type == "image"
|
||||
# Audio-output diffusion pipelines (those with ``support_audio_output = True``) now have
|
||||
# ``final_output_type="audio"`` set on the outer stage metadata as well as the inner request.
|
||||
assert first_output.final_output_type == "audio"
|
||||
assert hasattr(first_output, "request_output") and first_output.request_output
|
||||
|
||||
req_out = first_output.request_output
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online serving test for AudioX text-to-audio diffusion.
|
||||
|
||||
Mirrors `tests/e2e/online_serving/test_sd3_expansion.py` for image diffusion: spin up
|
||||
`vllm-omni` with `--model-class-name AudioXPipeline` and validate via the standard
|
||||
`send_diffusion_request` helper (now audio-aware).
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
# Tiny / random checkpoint usable in CI; override to a real bundle locally.
|
||||
AUDIOX_TEST_MODEL = os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")
|
||||
T2A_PROMPT = "A quiet living room with soft fabric rustle and gentle cat breathing."
|
||||
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"})
|
||||
|
||||
|
||||
def _audiox_server_cases(model: str):
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=["--model-class-name", "AudioXPipeline"],
|
||||
),
|
||||
id="t2a",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _audiox_server_cases(AUDIOX_TEST_MODEL), indirect=True)
|
||||
def test_audiox_t2a_online(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
"""AudioX text-to-audio: chat completion returns a non-empty WAV in `message.audio.data`."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": dummy_messages_from_mix_data(content_text=T2A_PROMPT),
|
||||
"extra_body": {
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 6.0,
|
||||
"seed": 42,
|
||||
"audiox_task": "t2a",
|
||||
"seconds_start": 0.0,
|
||||
"seconds_total": 2.0,
|
||||
},
|
||||
}
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -3,6 +3,7 @@
|
||||
import io
|
||||
import tempfile
|
||||
import threading
|
||||
import wave
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -115,8 +116,17 @@ def assert_audio_diffusion_response(
|
||||
) -> None:
|
||||
"""
|
||||
Validate audio diffusion response.
|
||||
|
||||
`response.audios` carries one entry per choice, each a `dict` with raw WAV
|
||||
bytes (`wav_bytes`) and the OpenAI audio metadata (`id`, `expires_at`).
|
||||
"""
|
||||
raise NotImplementedError("Audio validation is not implemented yet")
|
||||
assert response.audios, "Audio response is empty"
|
||||
for audio in response.audios:
|
||||
wav_bytes = audio.get("wav_bytes")
|
||||
assert wav_bytes, "Audio entry missing decoded WAV bytes"
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
|
||||
assert wav_file.getnframes() > 0, "Decoded WAV has zero frames"
|
||||
assert wav_file.getframerate() > 0, "Decoded WAV has invalid sample rate"
|
||||
|
||||
|
||||
def _maybe_int(value: Any) -> int | None:
|
||||
|
||||
@@ -701,6 +701,7 @@ class OpenAIClientHandler:
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
images = []
|
||||
audios = []
|
||||
for choice in chat_completion.choices:
|
||||
content = getattr(choice.message, "content", None)
|
||||
if isinstance(content, list):
|
||||
@@ -714,8 +715,21 @@ class OpenAIClientHandler:
|
||||
if image_url and image_url.startswith("data:image"):
|
||||
b64_data = image_url.split(",", 1)[1]
|
||||
images.append(decode_b64_image(b64_data))
|
||||
|
||||
# OpenAI audio responses (e.g. AudioX text-to-audio) populate `message.audio`.
|
||||
audio_obj = getattr(choice.message, "audio", None)
|
||||
audio_b64 = getattr(audio_obj, "data", None) if audio_obj is not None else None
|
||||
if audio_b64:
|
||||
audios.append(
|
||||
{
|
||||
"wav_bytes": base64.b64decode(audio_b64),
|
||||
"id": getattr(audio_obj, "id", None),
|
||||
"expires_at": getattr(audio_obj, "expires_at", None),
|
||||
}
|
||||
)
|
||||
result.e2e_latency = time.perf_counter() - start_time
|
||||
result.images = images if images else None
|
||||
result.audios = audios if audios else None
|
||||
result.success = True
|
||||
except Exception as e:
|
||||
result.error_message = f"Diffusion response processing error: {str(e)}"
|
||||
|
||||
Reference in new issue
Block a user