diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 1a6889663..cb80828eb 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -7,7 +7,7 @@ COPY . . # Install system dependencies RUN apt-get update && \ - apt-get install -y ffmpeg git sox libsox-fmt-all jq && \ + apt-get install -y espeak-ng ffmpeg git sox libsox-fmt-all jq && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index e9db9ee49..3e0b60650 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -19,7 +19,7 @@ WORKDIR ${COMMON_WORKDIR} # Step 1: Setup - Install system dependencies RUN apt-get update && \ - apt-get install -y ffmpeg git sox libsox-fmt-all jq && \ + apt-get install -y espeak-ng ffmpeg git sox libsox-fmt-all jq && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index 3d59f9b04..1595165e8 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -14,6 +14,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO RUN apt clean && apt-get update -y && \ apt-get install -y --no-install-recommends --fix-missing \ curl \ + espeak-ng \ ffmpeg \ git \ libsndfile1 \ diff --git a/pyproject.toml b/pyproject.toml index 99194c648..5bcfab7d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dev = [ "mooncake-transfer-engine==0.3.8.post1", "av", # for ComfyUI tests "openpyxl>=3.0.0", # for nightly CI + "pyttsx3>=2.99" ] docs = [ diff --git a/tests/conftest.py b/tests/conftest.py index c3ff10246..c775ae349 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ if "VLLM_TARGET_DEVICE" not in os.environ: import concurrent.futures import gc +import multiprocessing import socket import subprocess import sys @@ -217,102 +218,193 @@ def generate_synthetic_audio( sample_rate: int = 48000, # Default use 48000Hz. save_to_file: bool = False, ) -> dict[str, Any]: - """ "Generate synthetic audio with rain.""" + """ + Generate TTS speech with pyttsx3 and return base64 string. + """ + import tempfile + + import pyttsx3 import soundfile as sf - # Initialize audio data array - num_samples = int(sample_rate * duration) + def _pick_voice(engine: pyttsx3.Engine) -> str | None: + voices = engine.getProperty("voices") + if not voices: + return None + + preferred_tokens = ( + "natural", + "jenny", + "sonia", + "susan", + "zira", + "aria", + "hazel", + "samantha", + "ava", + "allison", + "female", + "woman", + "english-us", + "en-us", + "english", + ) + discouraged_tokens = ( + "espeak", + "robot", + "mbrola", + "microsoft david", + "male", + "man", + ) + + best_voice = voices[0] + best_score = float("-inf") + for voice in voices: + voice_text = f"{getattr(voice, 'id', '')} {getattr(voice, 'name', '')}".lower() + voice_languages = " ".join( + lang.decode(errors="ignore") if isinstance(lang, bytes) else str(lang) + for lang in getattr(voice, "languages", []) + ).lower() + combined_text = f"{voice_text} {voice_languages}" + score = 0 + for idx, token in enumerate(preferred_tokens): + if token in combined_text: + score += 20 - idx + for token in discouraged_tokens: + if token in combined_text: + score -= 10 + if "english" in combined_text or "en_" in combined_text or "en-" in combined_text: + score += 4 + if "en-us" in combined_text or "english-us" in combined_text: + score += 4 + if score > best_score: + best_score = score + best_voice = voice + + return best_voice.id + + def _resample_audio(audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray: + if src_sr == dst_sr or len(audio) == 0: + return audio.astype(np.float32) + + src_len = audio.shape[0] + dst_len = max(1, int(round(src_len * float(dst_sr) / float(src_sr)))) + src_idx = np.arange(src_len, dtype=np.float32) + dst_idx = np.linspace(0, src_len - 1, dst_len, dtype=np.float32) + + resampled_channels: list[np.ndarray] = [] + for ch in range(audio.shape[1]): + resampled_channels.append(np.interp(dst_idx, src_idx, audio[:, ch]).astype(np.float32)) + return np.stack(resampled_channels, axis=1) + + def _match_channels(audio: np.ndarray, target_channels: int) -> np.ndarray: + current_channels = audio.shape[1] + if current_channels == target_channels: + return audio.astype(np.float32) + if target_channels == 1: + return np.mean(audio, axis=1, keepdims=True, dtype=np.float32) + if current_channels == 1: + return np.repeat(audio, target_channels, axis=1).astype(np.float32) + + collapsed = np.mean(audio, axis=1, keepdims=True, dtype=np.float32) + return np.repeat(collapsed, target_channels, axis=1).astype(np.float32) + + def _trim_silence(audio: np.ndarray, threshold: float = 0.01) -> np.ndarray: + if len(audio) == 0: + return audio + energy = np.max(np.abs(audio), axis=1) + voiced = np.where(energy > threshold)[0] + if len(voiced) == 0: + return audio + start = max(0, int(voiced[0]) - int(sample_rate * 0.02)) + end = min(len(audio), int(voiced[-1]) + int(sample_rate * 0.04) + 1) + return audio[start:end] + + def _enhance_speech(audio: np.ndarray) -> np.ndarray: + if len(audio) == 0: + return audio.astype(np.float32) + enhanced = audio.astype(np.float32).copy() + enhanced -= np.mean(enhanced, axis=0, keepdims=True, dtype=np.float32) + if len(enhanced) > 1: + preemphasis = enhanced.copy() + preemphasis[1:] = enhanced[1:] - 0.94 * enhanced[:-1] + enhanced = 0.7 * enhanced + 0.3 * preemphasis + # Mild dynamic-range compression for ASR/TTS robustness. + enhanced = np.sign(enhanced) * np.sqrt(np.abs(enhanced)) + # Light fade to avoid clicks after trimming/repeating. + fade = min(len(enhanced) // 4, max(1, int(sample_rate * 0.01))) + if fade > 1: + ramp_in = np.linspace(0.0, 1.0, fade, dtype=np.float32) + ramp_out = np.linspace(1.0, 0.0, fade, dtype=np.float32) + enhanced[:fade] *= ramp_in[:, None] + enhanced[-fade:] *= ramp_out[:, None] + peak = float(np.max(np.abs(enhanced))) + if peak > 1e-8: + enhanced = enhanced / peak * 0.95 + return enhanced.astype(np.float32) + + phrase_text = "test" + num_samples = int(sample_rate * max(1, duration)) audio_data = np.zeros((num_samples, num_channels), dtype=np.float32) - # Configure parameters based on rain intensity - drop_density = 10 # Number of raindrops per second - drop_volume = 0.15 # Volume of individual raindrops - background_volume = 0.02 # Volume of background rain noise + engine = pyttsx3.init() + engine.setProperty("rate", 112) + engine.setProperty("volume", 1.0) + selected_voice = _pick_voice(engine) + if selected_voice is not None: + engine.setProperty("voice", selected_voice) - # Pink noise sounds more natural than white noise for rain - white_noise = np.random.randn(num_samples) - pink_noise = np.convolve(white_noise, np.ones(8) / 8, mode="same") - pink_noise = pink_noise / np.max(np.abs(pink_noise)) if np.max(np.abs(pink_noise)) > 0 else pink_noise - bg_noise = pink_noise * background_volume + temp_wav = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temp_wav.close() - # Add background noise to all channels - for ch in range(num_channels): - audio_data[:, ch] += bg_noise + try: + engine.save_to_file(phrase_text, temp_wav.name) + engine.runAndWait() + engine.stop() - # Total number of raindrops = density × duration × channels for stereo effect - total_drops = int(drop_density * duration * num_channels) + ready = False + for _ in range(50): + if os.path.exists(temp_wav.name) and os.path.getsize(temp_wav.name) > 44: + ready = True + break + time.sleep(0.1) - for _ in range(total_drops): - # Random timing for raindrop - drop_time = random.uniform(0, duration) + if not ready: + raise RuntimeError("pyttsx3 did not produce a WAV file in time.") - # Random duration of raindrop sound (0.01-0.05 seconds) - drop_duration = random.uniform(0.01, 0.05) + tts_audio, tts_sr = sf.read(temp_wav.name, dtype="float32", always_2d=True) + finally: + if os.path.exists(temp_wav.name): + os.unlink(temp_wav.name) - # Random frequency gives variation in raindrop pitch - drop_freq = random.uniform(500, 5000) # Hz + if len(tts_audio) == 0: + raise RuntimeError("pyttsx3 produced an empty WAV file.") - # Random channel selection for stereo positioning - channel = random.randint(0, num_channels - 1) + tts_audio = _resample_audio(tts_audio, tts_sr, sample_rate) + tts_audio = _match_channels(tts_audio, num_channels) + tts_audio = _trim_silence(tts_audio, threshold=0.012) + tts_audio = _enhance_speech(tts_audio) - # Calculate sample positions for this raindrop - start_sample = int(drop_time * sample_rate) - drop_samples = int(drop_duration * sample_rate) - end_sample = min(start_sample + drop_samples, num_samples) + lead_silence = min(int(sample_rate * 0.02), num_samples // 8) + pause_samples = int(sample_rate * 0.18) + start = lead_silence + phrase_len = tts_audio.shape[0] - if start_sample < end_sample: - # Generate the raindrop sound - num_drop_samples = end_sample - start_sample - t = np.arange(num_drop_samples) / sample_rate + while start < num_samples: + take = min(phrase_len, num_samples - start) + audio_data[start : start + take] = tts_audio[:take] + start += phrase_len + pause_samples - # Basic sine wave for raindrop sound - drop_sound = drop_volume * np.sin(2 * math.pi * drop_freq * t) - - # Apply envelope for natural attack and decay - envelope = np.ones(num_drop_samples) - attack_samples = int(num_drop_samples * 0.1) # 10% of samples for attack - decay_samples = num_drop_samples - attack_samples - - if attack_samples > 0: - # Linear attack: volume increases from 0 to 1 - envelope[:attack_samples] = np.linspace(0, 1, attack_samples) - - if decay_samples > 0: - # Exponential decay for natural sound fade - decay = np.exp(-8 * t[attack_samples:] / drop_duration) - envelope[attack_samples:] = decay - - # Apply envelope to raindrop sound - drop_sound *= envelope - - # Add raindrop sound to selected channel - audio_data[start_sample:end_sample, channel] += drop_sound - - # Step 3: Add simple reverb effect for realism - # Reverb simulates sound reflections in environment - if duration > 2: - # Single delay reverb (100ms delay) - delay_samples = int(0.1 * sample_rate) - if delay_samples < num_samples: - for ch in range(num_channels): - delayed = np.zeros(num_samples) - delayed[delay_samples:] = audio_data[:-delay_samples, ch] * 0.3 - audio_data[:, ch] += delayed - - # Step 4: Normalize audio to prevent clipping - # Find maximum amplitude and scale to 80% of maximum volume - max_amp = np.max(np.abs(audio_data)) + max_amp = float(np.max(np.abs(audio_data))) if max_amp > 0: - audio_data = audio_data / max_amp * 0.8 + audio_data = audio_data / max_amp * 0.95 - audio_array = audio_data.copy() - result = { - "np_array": audio_array, + audio_bytes: bytes | None = None + output_path: str | None = None + result: dict[str, Any] = { + "np_array": audio_data.copy(), } - # Handle file saving - audio_bytes = None - if save_to_file: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") output_path = f"audio_{num_channels}ch_{timestamp}.wav" @@ -617,24 +709,38 @@ def convert_audio_to_text(audio_data): return text -def convert_audio_file_to_text(output_path): +def _whisper_transcribe_in_current_process(output_path: str) -> str: import whisper - model = whisper.load_model("small") - text = model.transcribe( - output_path, - temperature=0.0, - word_timestamps=True, - condition_on_previous_text=False, - )["text"] - del model - if torch.cuda.is_available(): + device = "cuda" if torch.cuda.is_available() else "cpu" + model = whisper.load_model("small", device=device) + try: + text = model.transcribe( + output_path, + temperature=0.0, + word_timestamps=True, + condition_on_previous_text=False, + )["text"] + finally: + # Sync GPU so in-flight ops finish before we free the model; otherwise + # freed memory may not show up until those ops complete. + if torch is not None and torch.cuda.is_available(): + torch.cuda.synchronize() + del model gc.collect() - torch.cuda.empty_cache() - if text: - return text - else: - return "" + if torch is not None and torch.cuda.is_available(): + torch.cuda.empty_cache() + + return text or "" + + +def convert_audio_file_to_text(output_path: str) -> str: + """Convert an audio file to text in an isolated subprocess.""" + # Import locally to avoid impacting test module import time. + ctx = multiprocessing.get_context("spawn") + with concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=ctx) as executor: + future = executor.submit(_whisper_transcribe_in_current_process, output_path) + return future.result() def merge_base64_and_convert_to_text(base64_list): @@ -1105,7 +1211,7 @@ def assert_omni_response(response: OmniResponse, request_config: dict[str, Any], assert response.success, "The request failed." e2e_latency = response.e2e_latency if e2e_latency is not None: - print(f"the avg e2e is: {e2e_latency}") + print(f"the avg e2e latency is: {e2e_latency}") modalities = request_config.get("modalities", ["text", "audio"]) diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni.py b/tests/e2e/offline_inference/test_qwen2_5_omni.py index c5bb65842..d3c20580c 100644 --- a/tests/e2e/offline_inference/test_qwen2_5_omni.py +++ b/tests/e2e/offline_inference/test_qwen2_5_omni.py @@ -10,12 +10,29 @@ from tests.conftest import ( generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video, + modify_stage_config, ) from tests.utils import hardware_test from vllm_omni.platforms import current_omni_platform models = ["Qwen/Qwen2.5-Omni-7B"] + +def get_cuda_graph_config(): + path = modify_stage_config( + str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"), + updates={ + "stage_args": { + 0: { + "engine_args.enforce_eager": "true", + }, + 1: {"engine_args.enforce_eager": "true"}, + }, + }, + ) + return path + + # CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU if current_omni_platform.is_npu(): stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml") @@ -23,7 +40,7 @@ elif current_omni_platform.is_rocm(): # ROCm stage config optimized for MI325 GPU stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml") else: - stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml") + stage_config = get_cuda_graph_config() # Create parameter combinations for model and stage config test_params = [(model, stage_config) for model in models] diff --git a/tests/e2e/offline_inference/test_qwen3_omni.py b/tests/e2e/offline_inference/test_qwen3_omni.py index 9f76f911d..8c57becef 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni.py +++ b/tests/e2e/offline_inference/test_qwen3_omni.py @@ -13,18 +13,35 @@ import pytest from tests.conftest import ( generate_synthetic_video, + modify_stage_config, ) from tests.utils import hardware_test from vllm_omni.platforms import current_omni_platform models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] + +def get_cuda_graph_config(): + path = modify_stage_config( + str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"), + updates={ + "stage_args": { + 0: { + "engine_args.enforce_eager": "true", + }, + 1: {"engine_args.enforce_eager": "true"}, + }, + }, + ) + return path + + # CI stage config for 2xH100-80G GPUs or AMD GPU MI325 if current_omni_platform.is_rocm(): # ROCm stage config optimized for MI325 GPU stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")] else: - stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")] + stage_configs = [get_cuda_graph_config()] # Create parameter combinations for model and stage config test_params = [(model, stage_config) for model in models for stage_config in stage_configs] diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index 74eaa9c6b..fb9986592 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -114,7 +114,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: "messages": messages, "stream": True, "key_words": { - "audio": ["water", "chirping", "crackling", "rain"], + "audio": ["test"], "image": ["square", "quadrate"], }, } @@ -126,13 +126,14 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.advanced_model @pytest.mark.core_model @pytest.mark.omni +@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ Test text input processing and text/audio output generation via OpenAI API. Deploy Setting: default yaml Input Modal: text - Output Modal: text + audio + Output Modal: text Datasets: few requests """ messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index e2f77af73..fd7e0704c 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -22,7 +22,7 @@ from tests.utils import hardware_test models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] -AUDIO_KEY = ["water", "chirping", "crackling", "rain"] +AUDIO_KEY = ["test"] IMAGE_KEY = ["square", "quadrate"] VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"] @@ -90,12 +90,19 @@ def get_max_batch_size(size_type="few"): @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text + Output Modal: audio + Input Setting: stream=True + Datasets: single request + """ messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) request_config = { "model": omni_server.model, "messages": messages, "modalities": ["audio"], + "stream": True, "key_words": {"text": ["beijing"]}, } @@ -107,6 +114,12 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text + Output Modal: text, audio + Input Setting: stream=False + Datasets: few requests + """ messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) request_config = { @@ -115,7 +128,7 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None: "key_words": {"text": ["beijing"]}, } - openai_client.send_request(request_config) + openai_client.send_request(request_config, request_num=get_max_batch_size()) @pytest.mark.advanced_model @@ -123,6 +136,12 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_text_001(omni_server, openai_client) -> None: + """ + Input Modal: image + Output Modal: text + Input Setting: stream=True + Datasets: single request + """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" messages = dummy_messages_from_mix_data(image_data_url=image_data_url) @@ -130,6 +149,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None: "model": omni_server.model, "messages": messages, "modalities": ["text"], + "stream": True, "key_words": {"image": IMAGE_KEY}, } @@ -141,6 +161,12 @@ def test_image_to_text_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: image + Output Modal: audio + Input Setting: stream=False + Datasets: single request + """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" messages = dummy_messages_from_mix_data(image_data_url=image_data_url) @@ -159,6 +185,12 @@ def test_image_to_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: image + Output Modal: text, audio + Input Setting: stream=False + Datasets: few requests + """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(1280, 720)['base64']}" messages = dummy_messages_from_mix_data(image_data_url=image_data_url) @@ -177,6 +209,12 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_text_001(omni_server, openai_client) -> None: + """ + Input Modal: video + Output Modal: text + Input Setting: stream=False + Datasets: single request + """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" messages = dummy_messages_from_mix_data(video_data_url=video_data_url) @@ -195,6 +233,12 @@ def test_video_to_text_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: video + Output Modal: audio + Input Setting: stream=False + Datasets: single request + """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" messages = dummy_messages_from_mix_data(video_data_url=video_data_url) @@ -213,6 +257,12 @@ def test_video_to_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: video + Output Modal: text, audio + Input Setting: stream=False + Datasets: few requests + """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" messages = dummy_messages_from_mix_data(video_data_url=video_data_url) @@ -231,6 +281,12 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text, audio + Output Modal: text, audio + Input Setting: stream=False + Datasets: single request + """ audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" messages = dummy_messages_from_mix_data( audio_data_url=audio_data_url, system_prompt=get_system_prompt(), content_text=get_prompt("text_audio") @@ -239,7 +295,6 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None: request_config = { "model": omni_server.model, "messages": messages, - "modalities": ["text"], "key_words": {"audio": AUDIO_KEY}, } @@ -251,6 +306,12 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_image_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text, image + Output Modal: text, audio + Input Setting: stream=False + Datasets: single request + """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" messages = dummy_messages_from_mix_data( @@ -260,19 +321,23 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None: request_config = { "model": omni_server.model, "messages": messages, - "modalities": ["audio"], "key_words": {"image": IMAGE_KEY}, } openai_client.send_request(request_config) -@pytest.mark.skip(reason="There is a known issue with oom error.") @pytest.mark.advanced_model @pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text, video + Output Modal: text, audio + Input Setting: stream=True + Datasets: single requests + """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(1280, 720, 30)['base64']}" messages = dummy_messages_from_mix_data( @@ -282,10 +347,11 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None: request_config = { "model": omni_server.model, "messages": messages, + "stream": True, "key_words": {"video": VIDEO_KEY}, } - openai_client.send_request(request_config, request_num=get_max_batch_size()) + openai_client.send_request(request_config) @pytest.mark.skip(reason="There is a known issue with shape mismatch error.") @@ -294,6 +360,12 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: + """ + Input Modal: text, audio, image, video + Output Modal: text, audio + Input Setting: stream=True + Datasets: few requests + """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"