[CI] Add online e2e test for qwen2.5 omni (#1668)

Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com>
Signed-off-by: Junhong Liu <ljh_lbj@163.com>

.buildkite/test-merge.yml

@@ -194,6 +194,7 @@ steps:
       - export VLLM_LOGGING_LEVEL=DEBUG
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
       - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+      - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:

.buildkite/test-ready.yml

@@ -201,7 +201,7 @@ steps:
         timeout 17m bash -c '
           export VLLM_LOGGING_LEVEL=DEBUG
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+          pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "core_model" --run-level "core_model"
         '
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU

docs/contributing/ci/CI_5levels.md

@@ -225,6 +225,7 @@ vllm_omni/                                    tests/
 │
 └── e2e (tests side only)               →     ├── e2e/
                                                ├── online_serving/                     ✅ non-empty
+                                               │   ├── test_qwen2_5_omni.py
                                                │   ├── test_async_omni.py
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py

docs/contributing/ci/tests_style.md

@@ -130,6 +130,7 @@ vllm_omni/                                    tests/
 │
 └── e2e (tests side only)               →     ├── e2e/
                                                ├── online_serving/                     ✅ non-empty
+                                               │   ├── test_qwen2_5_omni.py
                                                │   ├── test_async_omni.py
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py

tests/e2e/online_serving/test_qwen2_5_omni.py

@@ -0,0 +1,137 @@
+"""
+E2E Online tests for Qwen2.5-Omni model with video input and audio output.
+"""
+
+import os
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import (
+    OmniServerParams,
+    dummy_messages_from_mix_data,
+    generate_synthetic_audio,
+    generate_synthetic_image,
+    generate_synthetic_video,
+    modify_stage_config,
+)
+from tests.utils import hardware_test
+from vllm_omni.platforms import current_omni_platform
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+
+models = ["Qwen/Qwen2.5-Omni-7B"]
+
+
+def get_config():
+    path = modify_stage_config(
+        str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
+    )
+    return path
+
+
+# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
+if current_omni_platform.is_rocm():
+    # ROCm stage config optimized for MI325 GPU
+    stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")]
+else:
+    stage_configs = [get_config()]
+
+# Create parameter combinations for model and stage config
+test_params = [
+    OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
+]
+
+
+def get_system_prompt():
+    return {
+        "role": "system",
+        "content": [
+            {
+                "type": "text",
+                "text": (
+                    "You are Qwen, a virtual human developed by the Qwen Team, "
+                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
+                    "as well as generating text and speech."
+                ),
+            }
+        ],
+    }
+
+
+def get_prompt(prompt_type="text_only"):
+    prompts = {
+        "text_only": "What is the capital of China? Answer in 20 words.",
+        "mix": "What is recited in the audio? What is in this image? Describe the video briefly.",
+    }
+    return prompts.get(prompt_type, prompts["text_only"])
+
+
+def get_max_batch_size(size_type="few"):
+    batch_sizes = {"few": 5, "medium": 100, "large": 256}
+    return batch_sizes.get(size_type, 5)
+
+
+@pytest.mark.advanced_model
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
+    """
+    Test multi-modal input processing and text/audio output generation via OpenAI API.
+    Deploy Setting: default yaml
+    Input Modal: text + audio + video + image
+    Output Modal: text + audio
+    Input Setting: stream=True
+    Datasets: single request
+    """
+
+    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
+    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
+    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1, sample_rate=16000)['base64']}"
+    messages = dummy_messages_from_mix_data(
+        system_prompt=get_system_prompt(),
+        video_data_url=video_data_url,
+        image_data_url=image_data_url,
+        audio_data_url=audio_data_url,
+        content_text=get_prompt("mix"),
+    )
+
+    request_config = {
+        "model": omni_server.model,
+        "messages": messages,
+        "stream": True,
+        "key_words": {
+            "video": ["sphere", "globe", "circle", "round", "ball"],
+        },
+    }
+
+    # Test single completion
+    openai_client.send_omni_request(request_config)
+
+
+@pytest.mark.advanced_model
+@pytest.mark.omni
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_text_001(omni_server, openai_client) -> None:
+    """
+    Test text input processing and text/audio output generation via OpenAI API.
+    Deploy Setting: default yaml
+    Input Modal: text
+    Output Modal: text
+    Datasets: few requests
+    """
+    messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt())
+
+    request_config = {
+        "model": omni_server.model,
+        "messages": messages,
+        "stream": False,
+        "modalities": ["text"],
+        "key_words": {"text": ["beijing"]},
+    }
+
+    openai_client.send_omni_request(request_config, request_num=get_max_batch_size())

tests/e2e/stage_configs/qwen2_5_omni_ci.yaml

@@ -22,6 +22,7 @@ stage_args:
       engine_output_type: latent
       enable_prefix_caching: false
       mm_processor_cache_gb: 0
+      load_format: dummy
     is_comprehension: true
     final_output: true
     final_output_type: text
@@ -51,6 +52,7 @@ stage_args:
       trust_remote_code: true
       enable_prefix_caching: false
       engine_output_type: latent
+      load_format: dummy
     engine_input_source: [0]
     custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
     default_sampling_params:
@@ -77,8 +79,9 @@ stage_args:
       trust_remote_code: true
       enable_prefix_caching: false
       engine_output_type: audio
-      max_num_batched_tokens: 4096
-      max_model_len: 4096
+      max_num_batched_tokens: 8192
+      max_model_len: 8192
+      load_format: dummy
     engine_input_source: [1]
     final_output: true
     final_output_type: audio
@@ -86,7 +89,7 @@ stage_args:
       temperature: 0.0
       top_p: 1.0
       top_k: -1
-      max_tokens: 4096
+      max_tokens: 8192
       seed: 42
       detokenize: True
       repetition_penalty: 1.1

vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py

@@ -113,6 +113,15 @@ class Qwen2_5OmniForConditionalGeneration(
                 self.model.set_suppress_start_id(t2w_token_end_id + 1)
             self.requires_raw_input_tokens = True
 
+            # Initialize thinker_embedding from config (random weights).
+            # For load_format: dummy this is the final state;
+            # for normal loading, load_weights() overwrites with real weights.
+            self.thinker_embedding = nn.Embedding(
+                self.thinker_config.text_config.vocab_size,
+                self.thinker_config.text_config.hidden_size,
+            )
+            self._init_special_tokens_embeddings()
+
         elif self.model_stage == "code2wav":
             self.thinker = None
             self.talker = None