[CI] Update nightly/ready test scheduling and unskip selected cases (#2945)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -141,5 +141,5 @@ docker run \
     "${image_name}" \
     bash -c '
     set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
 '

.buildkite/test-amd-merge.yml

@@ -111,8 +111,8 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2

.buildkite/test-amd-ready.yaml

@@ -112,19 +112,6 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
-
-# NOTE: This test is not running any thing. It is skipped and deselected.
-# Currently it is = 1 skipped, 1 deselected, 17 warnings in 0.03s ======
-# - label: "Omni Model Test Qwen2-5-Omni"
-#   agent_pool: mi325_2
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py -m "core_model" --run-level "core_model"
-
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi325_2
 #   depends_on: amd-build

.buildkite/test-amd.yaml

@@ -88,7 +88,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
     - pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
 - label: "Omni Model Test Qwen3-Omni"

.buildkite/test-merge.yml

@@ -172,25 +172,6 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "Omni Model Test"
-    timeout_in_minutes: 30
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: "Qwen3-TTS CustomVoice E2E Test"
     depends_on: upload-merge-pipeline
     commands:
@@ -385,7 +366,7 @@ steps:
           set -e
           export VLLM_TEST_CLEAN_GPU_MEMORY=1
           export VLLM_IMAGE_FETCH_TIMEOUT=60
-          pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory"
+          pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model"
           pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model"
         '
     agents:

.buildkite/test-nightly.yml

@@ -10,7 +10,7 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
     steps:
-      - label: ":full_moon: Omni · Function Test"
+      - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
@@ -48,6 +48,25 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+      - label: ":full_moon: Omni · Function Test with L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+        agents:
+          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
       - label: ":full_moon: Omni · Doc Test with L4"
         timeout_in_minutes: 90
         commands:
@@ -205,7 +224,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" -k "not test_qwen2_5" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:

tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py

@@ -17,6 +17,7 @@ import time
 import pytest
 import torch
 
+from tests.helpers.mark import hardware_test
 from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import (
     ManagedBuffer,
     MooncakeTransferEngineConnector,
@@ -24,7 +25,7 @@ from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_c
 )
 
 # All tests in this file require Mooncake TransferEngine and an RDMA environment.
-pytestmark = [pytest.mark.parallel, pytest.mark.gpu]
+pytestmark = [pytest.mark.parallel, pytest.mark.core_model]
 
 # ---------------------------------------------------------------------------
 # Shared helpers
@@ -194,7 +195,7 @@ def _md5(tensor: torch.Tensor) -> str:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestBasicConnector:
     """Verify connector initialization, put, cleanup, and health check."""
 
@@ -254,7 +255,7 @@ class TestBasicConnector:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestEndToEnd:
     """E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types."""
 
@@ -447,7 +448,7 @@ class TestEndToEnd:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestLifecycle:
     """Close, context manager, double-close safety."""
 
@@ -475,9 +476,7 @@ class TestLifecycle:
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.cuda
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestGPUPool:
     """GPU memory pool: initialization, put (CPU/GPU tensor), E2E transfer."""
 
@@ -521,12 +520,11 @@ class TestGPUPool:
 
 
 # ---------------------------------------------------------------------------
-# 5. Stress / Correctness tests (marked slow, skipped in quick CI)
+# 5. Stress / Correctness tests, slow
 # ---------------------------------------------------------------------------
 
 
-@pytest.mark.slow
-@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
 class TestStressCorrectness:
     """
     Slow but high-value regression tests: concurrent put+get with data

tests/e2e/offline_inference/test_bagel_text2img.py

@@ -304,11 +304,13 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
 @pytest.mark.core_model
 @pytest.mark.advanced_model
 @pytest.mark.diffusion
-@hardware_test(res={"cuda": "H100"})
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_bagel_text2img_mooncake_connector(run_level):
     """Test Bagel text2img with Mooncake connector for inter-stage communication."""
     if not _is_mooncake_master_available():
-        pytest.skip("mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)")
+        raise RuntimeError(
+            "mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)"
+        )
     MOONCAKE_HOST = "127.0.0.1"
     MOONCAKE_RPC_PORT = _find_free_port()
     MOONCAKE_HTTP_PORT = _find_free_port()

tests/e2e/online_serving/test_qwen2_5_omni_expansion.py

@@ -56,11 +56,7 @@ def get_max_batch_size(size_type="few"):
     return batch_sizes.get(size_type, 5)
 
 
-@pytest.mark.skip(
-    reason="qwen2.5-omni model cannot recognize the synthetic video and audio data, need to update the test data to enable this test case"
-)
 @pytest.mark.advanced_model
-@pytest.mark.core_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -469,7 +469,6 @@ def test_speaker_001(omni_server, openai_client) -> None:
 
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
-@pytest.mark.skip(reason="Known issue: occasional inaccuracy in voice recognition.")
 def test_speaker_002(omni_server, openai_client) -> None:
     """
     Input Modal: text only (one-word answer constraint).

tests/helpers/assertions.py

@@ -318,7 +318,7 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str:
         top = outputs[0]
         label = str(top.get("label", "")).lower()
         conf = float(top.get("score", 0.0))
-        if conf < 0.5:
+        if conf < 0.6:
             gender = "unknown"
         elif ("female" in label) or ("жен" in label):
             gender = "female"