diff --git a/.buildkite/scripts/hardware_ci/run_npu_test.sh b/.buildkite/scripts/hardware_ci/run_npu_test.sh index fbe24badb..492699c5f 100644 --- a/.buildkite/scripts/hardware_ci/run_npu_test.sh +++ b/.buildkite/scripts/hardware_ci/run_npu_test.sh @@ -141,5 +141,5 @@ docker run \ "${image_name}" \ bash -c ' set -e - VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py ' diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index ac52f60b3..9db6e7da6 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -111,8 +111,8 @@ steps: commands: - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py - - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model" + - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py + - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model" - label: "Omni Model Test Qwen3-Omni" agent_pool: mi325_2 diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/test-amd-ready.yaml index 30bbc7694..21843e6df 100644 --- a/.buildkite/test-amd-ready.yaml +++ b/.buildkite/test-amd-ready.yaml @@ -112,19 +112,6 @@ steps: pytest -s -v tests/engine/test_async_omni_engine_abort.py ' - -# NOTE: This test is not running any thing. It is skipped and deselected. -# Currently it is = 1 skipped, 1 deselected, 17 warnings in 0.03s ====== -# - label: "Omni Model Test Qwen2-5-Omni" -# agent_pool: mi325_2 -# depends_on: amd-build -# mirror_hardwares: [amdproduction] -# grade: Blocking -# commands: -# - export VLLM_LOGGING_LEVEL=DEBUG -# - export VLLM_WORKER_MULTIPROC_METHOD=spawn -# - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py -m "core_model" --run-level "core_model" - # - label: "Omni Model Test Qwen3-Omni" # agent_pool: mi325_2 # depends_on: amd-build diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 0b9a3f47a..8942b6407 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -88,7 +88,7 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py - pytest -s -v tests/engine/test_async_omni_engine_abort.py - label: "Omni Model Test Qwen3-Omni" diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 848297cec..7194a6e21 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -172,25 +172,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Omni Model Test" - timeout_in_minutes: 30 - depends_on: upload-merge-pipeline - commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model" - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Qwen3-TTS CustomVoice E2E Test" depends_on: upload-merge-pipeline commands: @@ -385,7 +366,7 @@ steps: set -e export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" + pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model" ' agents: diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index ce67b76d9..c68215db5 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -10,7 +10,7 @@ steps: depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - - label: ":full_moon: Omni · Function Test" + - label: ":full_moon: Omni · Function Test with H100" timeout_in_minutes: 90 commands: - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy @@ -48,6 +48,25 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Omni · Function Test with L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model" + agents: + queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: ":full_moon: Omni · Doc Test with L4" timeout_in_minutes: 90 commands: @@ -205,7 +224,7 @@ steps: timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy + - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" -k "not test_qwen2_5" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: diff --git a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py index 7b376da62..e5781f719 100755 --- a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py +++ b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_rdma.py @@ -17,6 +17,7 @@ import time import pytest import torch +from tests.helpers.mark import hardware_test from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import ( ManagedBuffer, MooncakeTransferEngineConnector, @@ -24,7 +25,7 @@ from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_c ) # All tests in this file require Mooncake TransferEngine and an RDMA environment. -pytestmark = [pytest.mark.parallel, pytest.mark.gpu] +pytestmark = [pytest.mark.parallel, pytest.mark.core_model] # --------------------------------------------------------------------------- # Shared helpers @@ -194,7 +195,7 @@ def _md5(tensor: torch.Tensor) -> str: # --------------------------------------------------------------------------- -@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available") +@hardware_test(res={"cuda": "L4"}, num_cards=1) class TestBasicConnector: """Verify connector initialization, put, cleanup, and health check.""" @@ -254,7 +255,7 @@ class TestBasicConnector: # --------------------------------------------------------------------------- -@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available") +@hardware_test(res={"cuda": "L4"}, num_cards=1) class TestEndToEnd: """E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types.""" @@ -447,7 +448,7 @@ class TestEndToEnd: # --------------------------------------------------------------------------- -@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available") +@hardware_test(res={"cuda": "L4"}, num_cards=1) class TestLifecycle: """Close, context manager, double-close safety.""" @@ -475,9 +476,7 @@ class TestLifecycle: # --------------------------------------------------------------------------- -@pytest.mark.cuda -@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available") -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@hardware_test(res={"cuda": "L4"}, num_cards=1) class TestGPUPool: """GPU memory pool: initialization, put (CPU/GPU tensor), E2E transfer.""" @@ -521,12 +520,11 @@ class TestGPUPool: # --------------------------------------------------------------------------- -# 5. Stress / Correctness tests (marked slow, skipped in quick CI) +# 5. Stress / Correctness tests, slow # --------------------------------------------------------------------------- -@pytest.mark.slow -@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available") +@hardware_test(res={"cuda": "L4"}, num_cards=1) class TestStressCorrectness: """ Slow but high-value regression tests: concurrent put+get with data diff --git a/tests/e2e/offline_inference/test_bagel_text2img.py b/tests/e2e/offline_inference/test_bagel_text2img.py index 65cd8425c..5ac6916d2 100644 --- a/tests/e2e/offline_inference/test_bagel_text2img.py +++ b/tests/e2e/offline_inference/test_bagel_text2img.py @@ -304,11 +304,13 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.diffusion -@hardware_test(res={"cuda": "H100"}) +@hardware_test(res={"cuda": "H100"}, num_cards=1) def test_bagel_text2img_mooncake_connector(run_level): """Test Bagel text2img with Mooncake connector for inter-stage communication.""" if not _is_mooncake_master_available(): - pytest.skip("mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)") + raise RuntimeError( + "mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)" + ) MOONCAKE_HOST = "127.0.0.1" MOONCAKE_RPC_PORT = _find_free_port() MOONCAKE_HTTP_PORT = _find_free_port() diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni.py b/tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py similarity index 100% rename from tests/e2e/offline_inference/test_qwen2_5_omni.py rename to tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py diff --git a/tests/e2e/online_serving/test_qwen2_5_omni.py b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py similarity index 95% rename from tests/e2e/online_serving/test_qwen2_5_omni.py rename to tests/e2e/online_serving/test_qwen2_5_omni_expansion.py index 8a1a8eb99..a325952c0 100644 --- a/tests/e2e/online_serving/test_qwen2_5_omni.py +++ b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py @@ -56,11 +56,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skip( - reason="qwen2.5-omni model cannot recognize the synthetic video and audio data, need to update the test data to enable this test case" -) @pytest.mark.advanced_model -@pytest.mark.core_model @pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 2ebf5c7e3..f1dc39e57 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -469,7 +469,6 @@ def test_speaker_001(omni_server, openai_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.skip(reason="Known issue: occasional inaccuracy in voice recognition.") def test_speaker_002(omni_server, openai_client) -> None: """ Input Modal: text only (one-word answer constraint). diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index 604b76b62..333c60406 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -318,7 +318,7 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str: top = outputs[0] label = str(top.get("label", "")).lower() conf = float(top.get("score", 0.0)) - if conf < 0.5: + if conf < 0.6: gender = "unknown" elif ("female" in label) or ("жен" in label): gender = "female"