""" E2E Online tests for Qwen2.5-Omni model with video input and audio output. """ import os import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" models = ["Qwen/Qwen2.5-Omni-7B"] # Single CI deploy YAML; rocm/xpu deltas are picked automatically via the # platforms: section in vllm_omni/deploy/ci/qwen2_5_omni.yaml. stage_configs = [modify_stage_config(get_deploy_config_path("ci/qwen2_5_omni.yaml"))] # Create parameter combinations for model and stage config test_params = [ OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs ] def get_system_prompt(): return { "role": "system", "content": [ { "type": "text", "text": ( "You are Qwen, a virtual human developed by the Qwen Team, " "Alibaba Group, capable of perceiving auditory and visual inputs, " "as well as generating text and speech." ), } ], } def get_prompt(prompt_type="text_only"): prompts = { "text_only": "What is the capital of China? Answer in 20 words.", "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", } return prompts.get(prompt_type, prompts["text_only"]) def get_max_batch_size(size_type="few"): batch_sizes = {"few": 5, "medium": 100, "large": 256} return batch_sizes.get(size_type, 5) @pytest.mark.slow @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: """ Test multi-modal input processing and text/audio output generation via OpenAI API. Deploy Setting: default yaml Input Modal: text + audio + video + image Output Modal: text + audio Input Setting: stream=True Datasets: single request """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1, sample_rate=16000)['base64']}" messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), video_data_url=video_data_url, image_data_url=image_data_url, audio_data_url=audio_data_url, content_text=get_prompt("mix"), ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, "key_words": { "video": ["sphere", "globe", "circle", "round", "ball"], }, } # Test single completion openai_client.send_omni_request(request_config) @pytest.mark.slow @pytest.mark.omni @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ Test text input processing and text/audio output generation via OpenAI API. Deploy Setting: default yaml Input Modal: text Output Modal: text Datasets: few requests """ messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) request_config = { "model": omni_server.model, "messages": messages, "stream": False, "modalities": ["text"], "key_words": {"text": ["beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size())