vllm-omni/tests/e2e/online_serving/test_dynin_omni_expansion.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Example online tests for Dynin-Omni model.
"""

import base64
import gc
import os
from io import BytesIO
from pathlib import Path

import numpy as np
import pytest
import soundfile as sf
from vllm.assets.image import ImageAsset

from tests import conftest as tests_conftest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test

os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"

MODEL = "snu-aidas/Dynin-Omni"
STAGE_CONFIG = str(Path(__file__).parent.parent / "stage_configs" / "dynin_omni_ci.yaml")
_WHISPER_SAMPLE_RATE_HZ = 16_000

T2I_PROMPT = "A high quality detailed living room interior photo."
T2S_PROMPT = "Please read this sentence naturally: Hello from Dynin-Omni online serving."
I2I_PROMPT = "Transform this outdoor nature boardwalk scene into a painting style with vivid colors."

TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, stage_init_timeout=600)]
_STAGE_COUNT = 3
_I2I_STAGE_SAMPLING = {"max_tokens": 1, "temperature": 0.0, "top_p": 1.0, "detokenize": False}


def _prepare_audio_waveform_for_whisper(audio_data: np.ndarray, samplerate: int) -> np.ndarray:
    """Normalize decoded audio into a mono 16 kHz float32 waveform for Whisper."""
    if samplerate <= 0:
        raise ValueError(f"Invalid audio sample rate: {samplerate}")

    waveform = np.asarray(audio_data, dtype=np.float32)
    if waveform.ndim == 0:
        raise ValueError("Audio waveform must have at least one dimension")
    if waveform.ndim > 1:
        waveform = np.mean(waveform, axis=1)
    if waveform.size == 0:
        raise ValueError("Empty audio waveform")

    if samplerate != _WHISPER_SAMPLE_RATE_HZ:
        target_num_samples = max(int(round(waveform.shape[0] * _WHISPER_SAMPLE_RATE_HZ / samplerate)), 1)
        source_positions = np.arange(waveform.shape[0], dtype=np.float64)
        target_positions = np.linspace(
            0.0,
            max(waveform.shape[0] - 1, 0),
            num=target_num_samples,
            dtype=np.float64,
        )
        waveform = np.interp(target_positions, source_positions, waveform).astype(np.float32)

    return np.ascontiguousarray(np.clip(waveform, -1.0, 1.0), dtype=np.float32)


def _convert_audio_bytes_to_text_without_ffmpeg(raw_bytes: bytes) -> str:
    """Dynin t2s keeps Whisper transcription local to this test module and avoids ffmpeg."""
    import whisper

    data, samplerate = sf.read(BytesIO(raw_bytes), dtype="float32", always_2d=True)
    audio_waveform = _prepare_audio_waveform_for_whisper(data, samplerate)

    model = whisper.load_model("small", device="cpu")
    try:
        transcript = model.transcribe(
            audio_waveform,
            temperature=0.0,
            word_timestamps=True,
            condition_on_previous_text=False,
        )["text"]
    finally:
        del model
        gc.collect()

    return transcript or ""


@pytest.fixture
def dynin_t2s_openai_client(openai_client, monkeypatch):
    monkeypatch.setattr(
        tests_conftest,
        "convert_audio_bytes_to_text",
        _convert_audio_bytes_to_text_without_ffmpeg,
    )
    return openai_client


def _build_t2i_messages(prompt: str) -> list[dict]:
    return [{"role": "user", "content": [{"type": "text", "text": f"<|t2i|> {prompt}"}]}]


def _build_t2s_messages(prompt: str) -> list[dict]:
    return [{"role": "user", "content": [{"type": "text", "text": f"<|t2s|> {prompt}"}]}]


def _build_i2i_messages(prompt: str) -> list[dict]:
    input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
    buffer = BytesIO()
    input_image.save(buffer, format="JPEG")
    image_b64 = base64.b64encode(buffer.getvalue()).decode("utf-8")
    return [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": f"<|i2i|> {prompt}"},
                {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}},
            ],
        }
    ]


@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
def test_send_i2i_request_001(omni_server, openai_client) -> None:
    request_config = {
        "model": omni_server.model,
        "messages": _build_i2i_messages(I2I_PROMPT),
        "modalities": ["image"],
        "extra_body": {
            "sampling_params_list": [dict(_I2I_STAGE_SAMPLING) for _ in range(_STAGE_COUNT)],
        },
    }
    openai_client.send_diffusion_request(request_config)


@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
def test_send_t2i_request_001(omni_server, openai_client) -> None:
    request_config = {
        "model": omni_server.model,
        "messages": _build_t2i_messages(T2I_PROMPT),
        "modalities": ["image"],
    }
    openai_client.send_diffusion_request(request_config)


@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
def test_send_t2s_request_001(omni_server, dynin_t2s_openai_client) -> None:
    request_config = {
        "model": omni_server.model,
        "messages": _build_t2s_messages(T2S_PROMPT),
        "modalities": ["audio"],
    }
    dynin_t2s_openai_client.send_omni_request(request_config)