diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index c33d7b4d1..e6912bd49 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -13,7 +13,7 @@ steps: - label: ":full_moon: Omni · Function Test" timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and H100 and omni" --run-level "advanced_model" + - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "mithril-h100-pool" plugins: @@ -52,7 +52,7 @@ steps: timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/examples/ -m "advanced_model and omni and L4" --run-level "advanced_model" + - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model" agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -70,7 +70,7 @@ steps: - label: ":full_moon: Omni · Doc Test with H100" timeout_in_minutes: 90 commands: - - pytest -s -v tests/examples/ -m "advanced_model and omni and H100" --run-level "advanced_model" + - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: @@ -160,7 +160,7 @@ steps: timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and L4 and omni" --run-level "advanced_model" + - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -233,7 +233,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100" timeout_in_minutes: 120 commands: - - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion and not test_wan_2_1_vace_expansion and not hunyuan" -m "advanced_model and diffusion and H100" --run-level "advanced_model" + - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "mithril-h100-pool" plugins: @@ -271,7 +271,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4" timeout_in_minutes: 60 commands: - - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion and not test_wan_2_1_vace_expansion and not hunyuan" -m "advanced_model and diffusion and L4" --run-level "advanced_model" + - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: @@ -290,7 +290,7 @@ steps: timeout_in_minutes: 60 commands: - export VLLM_TEST_CLEAN_GPU_MEMORY="1" - - pytest -s -v tests/examples/*/test_text_to_image.py -m "advanced_model and example and H100" --run-level "advanced_model" + - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: @@ -328,7 +328,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test" timeout_in_minutes: 60 commands: - - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level advanced_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1 + - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1 - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json" agents: queue: "mithril-h100-pool" @@ -367,7 +367,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test" timeout_in_minutes: 60 commands: - - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level advanced_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1 + - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1 - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv" - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json" agents: @@ -409,7 +409,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level advanced_model + - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model agents: queue: "mithril-h100-pool" plugins: @@ -514,7 +514,7 @@ steps: - label: ":full_moon: Diffusion X2V · Function Test" timeout_in_minutes: 90 commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "advanced_model" --run-level "advanced_model" + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model" --run-level "full_model" agents: queue: "mithril-h100-pool" plugins: @@ -552,7 +552,7 @@ steps: - label: ":full_moon: Diffusion X2V · Accuracy Test" timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m advanced_model --run-level advanced_model + - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model agents: queue: "mithril-h100-pool" plugins: diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 3ca1747fe..080f18885 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -120,23 +120,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Audio Generation Model Test" - depends_on: upload-ready-pipeline - commands: - - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Diffusion Cache Backend Test" depends_on: upload-ready-pipeline commands: diff --git a/benchmarks/accuracy/README.md b/benchmarks/accuracy/README.md index 0d73215b6..dbe20916a 100644 --- a/benchmarks/accuracy/README.md +++ b/benchmarks/accuracy/README.md @@ -23,5 +23,5 @@ Test guidance: - Local static/self-checks live in `tests/benchmarks/test_accuracy_bench_utils.py`. - End-to-end generation/evaluation should be validated in a remote GPU environment. In the current repo marker system there is `L4` but no `L5` - marker, so benchmark smoke tests should be wired as `advanced_model + - benchmark + L4` when GPU capacity is available. + marker, so benchmark smoke tests should be wired as `full_model + + benchmark + L4` for nightly when GPU capacity is available. diff --git a/benchmarks/accuracy/image_to_image/README.md b/benchmarks/accuracy/image_to_image/README.md index ee1d58f10..86e7b0cf3 100644 --- a/benchmarks/accuracy/image_to_image/README.md +++ b/benchmarks/accuracy/image_to_image/README.md @@ -99,5 +99,5 @@ Notes: - This flow requires the optional Hugging Face `datasets` package. - `generate` writes `generation_manifest.json` with local output coverage. - The current repo marker set exposes `L4` but not `L5`, so if you promote an - end-to-end smoke test into CI, use the existing `advanced_model`, `benchmark`, - and `L4` markers or introduce a new repo-wide marker explicitly first. + end-to-end smoke test into CI, use the `full_model`, `benchmark`, + and `L4` markers for nightly (or `advanced_model` for merge) or introduce a new repo-wide marker explicitly first. diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index 2452ef5d4..7bad897e7 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -418,13 +418,13 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Explanation**: - @pytest.mark.advanced_model: Marks the test as L3 or L4 level, indicating that this test case performs deep validation, using real models for performance, integration, and accuracy testing. This forms a "basic-advanced" correspondence with the core_model mark at the L2 level. + @pytest.mark.advanced_model: Marks the test as L3 merge level, indicating deep validation with real models. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples). @pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly. @pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing. - **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation run logic at the merge or nightly level, you can mark it only with @pytest.mark.advanced_model. If you believe the test case needs to accommodate both basic run and deep validation test logic, you should mark it with both @pytest.mark.core_model and @pytest.mark.advanced_model. + **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation at merge (L3), use @pytest.mark.advanced_model. For L4 nightly-only expansion and doc-example tests, use @pytest.mark.full_model with `--run-level full_model`. If the test case needs both basic run and deep validation, mark with @pytest.mark.core_model and the appropriate L3/L4 marker (`advanced_model` and/or `full_model`). **2.4.2 Test Function Definition and Documentation** @@ -516,9 +516,11 @@ L3 level testing executes after code is merged into the main branch. Its core pu **Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n. - **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model performs deep validation. + **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model and full_model perform deep validation. -- ***Run Command***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model` +- ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model` + +- ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model` ## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing diff --git a/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md b/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md index ab4deecd6..e1309b1ad 100644 --- a/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md @@ -37,7 +37,7 @@ Currently all the features are available in online serving mode. Hence, only nee **Code Style** - Validation: test that the multimodal output files of your model have the correct shapes. `OpenAIClientHandler.send_diffusion_request` should have taken care of this. -- Test marks: always add `advanced_model` and `diffusion`. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/). +- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/). - To maximize code reuse, you may refer to - `tests/conftest.py` for `omni_server` (running server in subprocess) and `openai_client` fixtures (sending requests and validating output), `generate_synthetic_image` and `assert_XXX_valid` helper. - `tests/helpers/mark.py` for `@hardware_test(...)` and `hardware_marks`. diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index 08b2e3b4e..0c9ec2fb3 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -42,31 +42,46 @@ Our test scripts use the pytest framework. First, please use `git clone https:// ``` The latest test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml). -=== "L3 level & L4 level" +=== "L3 level" ```bash pytest -s -v -m "advanced_model" --run-level=advanced_model ``` - If you only want to run L3 test case, you can use: - ```bash - pytest -s -v e2e/ --ignore-glob='*expansion.py' -m "advanced_model" --run-level=advanced_model - ``` If you only want to run a specific test case, you can use: ```bash pytest -s -v test_xxxx.py --run-level=advanced_model ``` If you only want to run specific test cases on a particular platform, you can use: ```bash - pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model + pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model + ``` + The latest L3 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml). + + +=== "L4 level" + + ```bash + cd tests + pytest -s -v -m "full_model" --run-level=full_model + ``` + If you only want to run a specific test case, you can use: + ```bash + pytest -s -v test_xxxx.py --run-level=full_model + ``` + If you only want to run specific test cases on a particular platform, you can use: + ```bash + pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model ``` Note: To run performance tests (defaults to ``test_qwen_omni.json``; use ``--test-config-file tests/dfx/perf/tests/test_tts.json`` for TTS): ```bash pytest -s -v tests/dfx/perf/scripts/run_benchmark.py ``` + The latest L4 (nightly) test commands use the `full_model` marker and `--run-level full_model` (see [test-nightly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml) and [test-nightly-diffusion.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly-diffusion.yml)). Example: - The latest L3 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml). - - The latest L4 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml). + ```bash + cd tests + pytest -s -v -m "full_model and omni and H100" --run-level=full_model + ``` You can find more information about markers in the documentation: [marker doc](./tests_markers.md) diff --git a/docs/contributing/ci/tests_markers.md b/docs/contributing/ci/tests_markers.md index 7628db284..6130541a6 100644 --- a/docs/contributing/ci/tests_markers.md +++ b/docs/contributing/ci/tests_markers.md @@ -8,7 +8,8 @@ Defined in `pyproject.toml`: | Marker | Description | | ------------------ | --------------------------------------------------------- | | `core_model` | L1&L2 tests (run in each PR) | -| `advanced_model` | L3&L4 level tests (run in each merge or nightly) | +| `advanced_model` | L3 tests (run on each merge to main) | +| `full_model` | L4 tests (run nightly) | | `diffusion` | Diffusion model tests | | `omni` | Omni model tests | | `cache` | Cache backend tests | diff --git a/pyproject.toml b/pyproject.toml index b6f4092fd..0958b9395 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -179,7 +179,8 @@ addopts = [ markers = [ # ci/cd required "core_model: L1&L2 tests (run in each PR)", - "advanced_model: L3&L4 level tests (run in each merge or nightly)", + "advanced_model: L3 level tests (run on each merge)", + "full_model: L4 level tests (run nightly)", # function module markers "diffusion: Diffusion model tests", "omni: Omni model tests", diff --git a/tests/benchmarks/test_accuracy_bench_utils.py b/tests/benchmarks/test_accuracy_bench_utils.py index a0479fb1b..8425804ae 100644 --- a/tests/benchmarks/test_accuracy_bench_utils.py +++ b/tests/benchmarks/test_accuracy_bench_utils.py @@ -6,6 +6,9 @@ from pathlib import Path import pytest from PIL import Image +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) @@ -37,8 +40,6 @@ from benchmarks.accuracy.text_to_image.gbench import ( summarize_gebench_results, ) -pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] - def test_summarize_gebench_generated_records_groups_by_type(): records = [ diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 13011b4bd..f14638c15 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -17,6 +17,9 @@ from tests.dfx.conftest import ( ) from tests.helpers.runtime import OmniServer +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" @@ -293,6 +296,7 @@ def assert_result( print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}") +@pytest.mark.benchmark @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) def test_performance_benchmark(omni_server, benchmark_params): diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 23efa8bb0..d41f80dc7 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -32,6 +32,8 @@ from typing import Any, cast import psutil import pytest +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") @@ -662,8 +664,7 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: # --------------------------------------------------------------------------- # Test entry point # --------------------------------------------------------------------------- - - +@pytest.mark.benchmark @pytest.mark.parametrize( "diffusion_server", server_params, diff --git a/tests/diffusion/models/glm_image/test_glm_image_sp.py b/tests/diffusion/models/glm_image/test_glm_image_sp.py index 40d1c8730..06a1a116d 100644 --- a/tests/diffusion/models/glm_image/test_glm_image_sp.py +++ b/tests/diffusion/models/glm_image/test_glm_image_sp.py @@ -6,6 +6,8 @@ import pytest from vllm_omni.diffusion.data import DiffusionParallelConfig +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + @pytest.fixture(scope="function", autouse=True) def setup_sp_groups(mocker): @@ -24,9 +26,6 @@ def setup_sp_groups(mocker): yield -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - def test_glm_image_sp_plan_defined(): """Test that _sp_plan is properly defined on GlmImageTransformer2DModel.""" from vllm_omni.diffusion.models.glm_image.glm_image_transformer import ( diff --git a/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py b/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py index f095dfd49..0ba19c7ff 100644 --- a/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py +++ b/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py @@ -17,8 +17,6 @@ from vllm_omni.distributed.omni_coordinator import ( pytestmark = [pytest.mark.core_model, pytest.mark.cpu] -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]: ctx = zmq.Context.instance() diff --git a/tests/e2e/accuracy/test_gebench_h100_smoke.py b/tests/e2e/accuracy/test_gebench_h100_smoke.py index 748919269..2702710e4 100644 --- a/tests/e2e/accuracy/test_gebench_h100_smoke.py +++ b/tests/e2e/accuracy/test_gebench_h100_smoke.py @@ -9,10 +9,10 @@ from benchmarks.accuracy.text_to_image.gbench import main as gbench_main from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir from tests.helpers.mark import hardware_test +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_gebench_h100_smoke( gebench_accuracy_servers, diff --git a/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py b/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py index 6227d6368..789f7ec93 100644 --- a/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py +++ b/tests/e2e/accuracy/test_gedit_bench_h100_smoke.py @@ -10,10 +10,10 @@ from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir from tests.helpers.mark import hardware_test +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_gedit_bench_h100_smoke( gedit_accuracy_servers, diff --git a/tests/e2e/accuracy/test_qwen_image.py b/tests/e2e/accuracy/test_qwen_image.py index 8922d9d10..4b8215d54 100644 --- a/tests/e2e/accuracy/test_qwen_image.py +++ b/tests/e2e/accuracy/test_qwen_image.py @@ -17,6 +17,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + + MODEL_ID = "Qwen/Qwen-Image" MODEL_ENV_VAR = "QWEN_IMAGE_MODEL" PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." @@ -99,9 +102,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image: run_post_test_cleanup(enable_force=True) -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None: model = _model_name() diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index e17aca6e9..07deecca9 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -15,6 +15,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + + SINGLE_MODEL = "Qwen/Qwen-Image-Edit" MULTIPLE_MODEL = "Qwen/Qwen-Image-Edit-2509" WIDTH = 512 @@ -169,9 +172,7 @@ def _diffusers_output_multiple_image( ) -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_single_matches_diffusers( accuracy_artifact_root: Path, @@ -196,9 +197,7 @@ def test_qwen_image_edit_single_matches_diffusers( ) -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) @pytest.mark.skip( reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged." diff --git a/tests/e2e/accuracy/test_qwen_image_layered.py b/tests/e2e/accuracy/test_qwen_image_layered.py index 0ab9cb323..30ad2966f 100644 --- a/tests/e2e/accuracy/test_qwen_image_layered.py +++ b/tests/e2e/accuracy/test_qwen_image_layered.py @@ -17,6 +17,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + + MODEL_ID = "Qwen/Qwen-Image-Layered" MODEL_ENV_VAR = "QWEN_IMAGE_LAYERED_MODEL" PROMPT = "decompose into layers" @@ -126,9 +129,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o run_post_test_cleanup(enable_force=True) -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_layered_matches_diffusers(accuracy_artifact_root: Path, qwen_bear_image: Image.Image) -> None: model = _model_name() diff --git a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py index 3aa5da85c..1caef3bff 100644 --- a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py +++ b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py @@ -50,6 +50,8 @@ from tests.e2e.accuracy.wan22_i2v.wan22_i2v_video_similarity_common import ( from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + def test_parse_video_metadata_extracts_dimensions_and_fps() -> None: payload = { @@ -537,9 +539,7 @@ def _generate_offline_video(*, image_source: str) -> tuple[Path, Path]: return offline_path, offline_metadata_path -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_wan22_i2v_diffusers_offline_generates_video( wan22_i2v_image_source: str | None, @@ -563,9 +563,7 @@ def test_wan22_i2v_diffusers_offline_generates_video( assert offline_metadata["frame_count"] == NUM_FRAMES -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=2) @pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) def test_wan22_i2v_online_serving_generates_video( @@ -594,9 +592,7 @@ def test_wan22_i2v_online_serving_generates_video( assert online_metadata["frame_count"] == NUM_FRAMES -@pytest.mark.advanced_model @pytest.mark.benchmark -@pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}, num_cards=2) def test_wan22_i2v_serving_matches_diffusers_video_similarity( wan22_i2v_image_source: str | None, diff --git a/tests/e2e/offline_inference/test_quantization_fp8.py b/tests/e2e/offline_inference/test_quantization_fp8.py index 92cf351e3..fce7bfa27 100644 --- a/tests/e2e/offline_inference/test_quantization_fp8.py +++ b/tests/e2e/offline_inference/test_quantization_fp8.py @@ -27,9 +27,6 @@ Usage: """ import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - from pathlib import Path from typing import Any @@ -44,6 +41,8 @@ from vllm_omni.platforms import current_omni_platform pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + # ─── helpers ────────────────────────────────────────────────────────────────── diff --git a/tests/e2e/offline_inference/test_stable_audio_expansion.py b/tests/e2e/offline_inference/test_stable_audio_expansion.py index 2fc07eb24..a5d3e6d22 100644 --- a/tests/e2e/offline_inference/test_stable_audio_expansion.py +++ b/tests/e2e/offline_inference/test_stable_audio_expansion.py @@ -22,6 +22,8 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.platforms import current_omni_platform +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + _SAMPLE_RATE = 44100 _CLIP_DURATION_S = 2.0 @@ -72,8 +74,6 @@ def generate_stable_audio_short_clip( return audio -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.cache @hardware_test(res={"cuda": "L4", "xpu": "B60"}) def test_stable_audio_quantization_and_teacache() -> None: diff --git a/tests/e2e/online_serving/test_bagel_expansion.py b/tests/e2e/online_serving/test_bagel_expansion.py index aa289c6d9..21142439b 100644 --- a/tests/e2e/online_serving/test_bagel_expansion.py +++ b/tests/e2e/online_serving/test_bagel_expansion.py @@ -19,6 +19,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution." NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark" @@ -112,8 +114,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("ByteDance-Seed/BAGEL-7B-MoT"), diff --git a/tests/e2e/online_serving/test_dynin_omni_expansion.py b/tests/e2e/online_serving/test_dynin_omni_expansion.py index b63fcbd75..da179dbc8 100644 --- a/tests/e2e/online_serving/test_dynin_omni_expansion.py +++ b/tests/e2e/online_serving/test_dynin_omni_expansion.py @@ -5,29 +5,26 @@ Example online tests for Dynin-Omni model. """ import base64 -import gc import os from io import BytesIO from pathlib import Path -import numpy as np import pytest -import soundfile as sf from vllm.assets.image import ImageAsset from tests.helpers.mark import hardware_test -from tests.helpers.media import convert_audio_bytes_to_text from tests.helpers.runtime import OmniServerParams +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "snu-aidas/Dynin-Omni" STAGE_CONFIG = str(Path(__file__).parent.parent / "stage_configs" / "dynin_omni_ci.yaml") -_WHISPER_SAMPLE_RATE_HZ = 16_000 T2I_PROMPT = "A high quality detailed living room interior photo." -T2S_PROMPT = "Please read this sentence naturally: Hello from Dynin-Omni online serving." +T2S_PROMPT = "Please read this sentence naturally: Hello from online serving." I2I_PROMPT = "Transform this outdoor nature boardwalk scene into a painting style with vivid colors." TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, stage_init_timeout=600)] @@ -35,65 +32,6 @@ _STAGE_COUNT = 3 _I2I_STAGE_SAMPLING = {"max_tokens": 1, "temperature": 0.0, "top_p": 1.0, "detokenize": False} -def _prepare_audio_waveform_for_whisper(audio_data: np.ndarray, samplerate: int) -> np.ndarray: - """Normalize decoded audio into a mono 16 kHz float32 waveform for Whisper.""" - if samplerate <= 0: - raise ValueError(f"Invalid audio sample rate: {samplerate}") - - waveform = np.asarray(audio_data, dtype=np.float32) - if waveform.ndim == 0: - raise ValueError("Audio waveform must have at least one dimension") - if waveform.ndim > 1: - waveform = np.mean(waveform, axis=1) - if waveform.size == 0: - raise ValueError("Empty audio waveform") - - if samplerate != _WHISPER_SAMPLE_RATE_HZ: - target_num_samples = max(int(round(waveform.shape[0] * _WHISPER_SAMPLE_RATE_HZ / samplerate)), 1) - source_positions = np.arange(waveform.shape[0], dtype=np.float64) - target_positions = np.linspace( - 0.0, - max(waveform.shape[0] - 1, 0), - num=target_num_samples, - dtype=np.float64, - ) - waveform = np.interp(target_positions, source_positions, waveform).astype(np.float32) - - return np.ascontiguousarray(np.clip(waveform, -1.0, 1.0), dtype=np.float32) - - -def _convert_audio_bytes_to_text_without_ffmpeg(raw_bytes: bytes) -> str: - """Dynin t2s keeps Whisper transcription local to this test module and avoids ffmpeg.""" - import whisper - - data, samplerate = sf.read(BytesIO(raw_bytes), dtype="float32", always_2d=True) - audio_waveform = _prepare_audio_waveform_for_whisper(data, samplerate) - - model = whisper.load_model("small", device="cpu") - try: - transcript = model.transcribe( - audio_waveform, - temperature=0.0, - word_timestamps=True, - condition_on_previous_text=False, - )["text"] - finally: - del model - gc.collect() - - return transcript or "" - - -@pytest.fixture -def dynin_t2s_openai_client(openai_client, monkeypatch): - monkeypatch.setattr( - convert_audio_bytes_to_text, - "convert_audio_bytes_to_text", - _convert_audio_bytes_to_text_without_ffmpeg, - ) - return openai_client - - def _build_t2i_messages(prompt: str) -> list[dict]: return [{"role": "user", "content": [{"type": "text", "text": f"<|t2i|> {prompt}"}]}] @@ -118,8 +56,6 @@ def _build_i2i_messages(prompt: str) -> list[dict]: ] -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) def test_send_i2i_request_001(omni_server, openai_client) -> None: @@ -134,8 +70,6 @@ def test_send_i2i_request_001(omni_server, openai_client) -> None: openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) def test_send_t2i_request_001(omni_server, openai_client) -> None: @@ -147,14 +81,13 @@ def test_send_t2i_request_001(omni_server, openai_client) -> None: openai_client.send_diffusion_request(request_config) -@pytest.mark.core_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) -def test_send_t2s_request_001(omni_server, dynin_t2s_openai_client) -> None: +def test_send_t2s_request_001(omni_server, openai_client) -> None: request_config = { "model": omni_server.model, "messages": _build_t2s_messages(T2S_PROMPT), "modalities": ["audio"], + "audio_ref_text": T2S_PROMPT, } - dynin_t2s_openai_client.send_omni_request(request_config) + openai_client.send_omni_request(request_config) diff --git a/tests/e2e/online_serving/test_flux2_expansion.py b/tests/e2e/online_serving/test_flux2_expansion.py index ce06ad564..9a2b164b3 100644 --- a/tests/e2e/online_serving/test_flux2_expansion.py +++ b/tests/e2e/online_serving/test_flux2_expansion.py @@ -14,6 +14,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4) POSITIVE_PROMPT = "A cat sitting on a windowsill" NEGATIVE_PROMPT = "blurry, low quality" @@ -86,8 +88,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases( diff --git a/tests/e2e/online_serving/test_flux2_klein_inpaint_expansion.py b/tests/e2e/online_serving/test_flux2_klein_inpaint_expansion.py index 0c45bb33f..a3d4f004e 100644 --- a/tests/e2e/online_serving/test_flux2_klein_inpaint_expansion.py +++ b/tests/e2e/online_serving/test_flux2_klein_inpaint_expansion.py @@ -16,6 +16,8 @@ from PIL import Image, ImageDraw from tests.helpers.runtime import OmniServer, OmniServerParams +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + MODEL = "black-forest-labs/FLUX.2-klein-4B" _HEIGHT = 512 @@ -66,8 +68,6 @@ def _send_edit_request(host: str, port: int, model: str, image_b64: str, mask_b6 return response.json() -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_flux2_klein_inpaint_basic(omni_server: OmniServer): input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128))) @@ -92,8 +92,6 @@ def test_flux2_klein_inpaint_basic(omni_server: OmniServer): assert img.size == (_WIDTH, _HEIGHT) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_flux2_klein_inpaint_deterministic(omni_server: OmniServer): input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128))) @@ -137,8 +135,6 @@ def test_flux2_klein_inpaint_deterministic(omni_server: OmniServer): ) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) def test_flux2_klein_inpaint_multiple_outputs(omni_server: OmniServer): input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128))) diff --git a/tests/e2e/online_serving/test_flux_2_dev_expansion.py b/tests/e2e/online_serving/test_flux_2_dev_expansion.py index addf6f002..953cb448a 100644 --- a/tests/e2e/online_serving/test_flux_2_dev_expansion.py +++ b/tests/e2e/online_serving/test_flux_2_dev_expansion.py @@ -17,6 +17,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + MODEL = "black-forest-labs/FLUX.2-dev" PROMPT = "A cinematic mountain landscape at sunrise, dramatic clouds, ultra-detailed, realistic photography." NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark" @@ -54,8 +56,6 @@ def _get_flux_2_dev_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_flux_2_dev_feature_cases(MODEL), diff --git a/tests/e2e/online_serving/test_flux_kontext_expansion.py b/tests/e2e/online_serving/test_flux_kontext_expansion.py index 574bb3db4..fd7d7d1b4 100644 --- a/tests/e2e/online_serving/test_flux_kontext_expansion.py +++ b/tests/e2e/online_serving/test_flux_kontext_expansion.py @@ -8,6 +8,8 @@ import pytest from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist" MODEL = "black-forest-labs/FLUX.1-Kontext-dev" @@ -29,8 +31,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), @@ -54,8 +54,6 @@ def test_flux_kontext_text_to_image(omni_server: OmniServer, openai_client: Open openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), @@ -83,8 +81,6 @@ def test_flux_kontext_image_edit(omni_server: OmniServer, openai_client: OpenAIC openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), @@ -110,8 +106,6 @@ def test_flux_kontext_image_edit_no_negative(omni_server: OmniServer, openai_cli openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), @@ -135,8 +129,6 @@ def test_flux_kontext_high_resolution(omni_server: OmniServer, openai_client: Op openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), diff --git a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py index 7af4c4344..681d8ff23 100644 --- a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py +++ b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py @@ -14,6 +14,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion." NEGATIVE_PROMPT = "low quality, blurry, distorted" @@ -62,8 +64,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(MODEL), diff --git a/tests/e2e/online_serving/test_longcat_image_edit_expansion.py b/tests/e2e/online_serving/test_longcat_image_edit_expansion.py index 9a96280f2..4be5fe42d 100644 --- a/tests/e2e/online_serving/test_longcat_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_longcat_image_edit_expansion.py @@ -17,6 +17,8 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting." NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) @@ -50,8 +52,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"), diff --git a/tests/e2e/online_serving/test_longcat_image_expansion.py b/tests/e2e/online_serving/test_longcat_image_expansion.py index 7d12db6a2..b9fd858b0 100644 --- a/tests/e2e/online_serving/test_longcat_image_expansion.py +++ b/tests/e2e/online_serving/test_longcat_image_expansion.py @@ -16,6 +16,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + TEXT_TO_IMAGE_PROMPT = ( "A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed." ) @@ -51,8 +53,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("meituan-longcat/LongCat-Image"), diff --git a/tests/e2e/online_serving/test_nextstep_expansion.py b/tests/e2e/online_serving/test_nextstep_expansion.py index 004a0967b..75290db5d 100644 --- a/tests/e2e/online_serving/test_nextstep_expansion.py +++ b/tests/e2e/online_serving/test_nextstep_expansion.py @@ -14,6 +14,8 @@ from tests.helpers.runtime import ( dummy_messages_from_mix_data, ) +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + # L4: 4 GPUs + TP=4; XPU B60: 2 cards (use num_cards={"cuda": 4, "xpu": 4} if needed) FOUR_CARD_MARKS = hardware_marks( res={"cuda": "L4", "xpu": "B60"}, @@ -45,8 +47,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(model=os.environ.get("VLLM_TEST_NEXTSTEP_MODEL", _DEFAULT_MODEL)), diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 6be2ebb2a..1388f7543 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -6,8 +6,6 @@ E2E Online tests for Qwen3-Omni model. import os -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - import pytest from tests.helpers.mark import hardware_test @@ -15,6 +13,10 @@ from tests.helpers.media import generate_synthetic_audio, generate_synthetic_ima from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + model = "Qwen/Qwen3-Omni-30B-A3B-Instruct" AUDIO_KEY = ["test"] @@ -130,8 +132,6 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: @@ -154,8 +154,6 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_text_to_text_audio_001(omni_server, openai_client) -> None: @@ -176,8 +174,6 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_001(omni_server, openai_client) -> None: @@ -202,8 +198,6 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None: @@ -227,8 +221,6 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_text_audio_to_text_audio_002(omni_server, openai_client) -> None: @@ -254,8 +246,6 @@ def test_text_audio_to_text_audio_002(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_text_image_to_text_audio_001(omni_server, openai_client) -> None: @@ -280,8 +270,6 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_large_image_to_text_audio_001(omni_server, openai_client) -> None: @@ -310,8 +298,6 @@ def test_large_image_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_text_video_to_text_audio_001(omni_server, openai_client) -> None: @@ -338,8 +324,6 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.skip(reason="There is a known issue with shape mismatch error.") -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: @@ -369,8 +353,6 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.skip(reason="issue: #2827") @@ -398,8 +380,6 @@ def test_audio_in_video_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_in_video_002(omni_server, openai_client) -> None: @@ -427,8 +407,6 @@ def test_audio_in_video_002(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_one_word_prompt_001(omni_server, openai_client) -> None: @@ -463,8 +441,6 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None: print(f"Similarity assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}") -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_speaker_001(omni_server, openai_client) -> None: @@ -490,8 +466,6 @@ def test_speaker_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.skip(reason="Known issue: occasional inaccuracy in voice recognition.") @@ -528,8 +502,6 @@ def test_speaker_002(omni_server, openai_client) -> None: print(f"Gender assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}") -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_speaker_003(omni_server, openai_client) -> None: @@ -555,8 +527,6 @@ def test_speaker_003(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_language_001(omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py index 527eeaef4..d86f96af0 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py @@ -9,15 +9,17 @@ actual model inference, not mocks. import os -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" - import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav" @@ -61,9 +63,6 @@ tts_server_params = [ ] -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_clone_streaming_001(omni_server, openai_client) -> None: @@ -89,9 +88,6 @@ def test_voice_clone_streaming_001(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size("few")) -@pytest.mark.advanced_model -@pytest.mark.core_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_response_format_001(omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py index b32303842..d97e70e41 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py @@ -9,15 +9,17 @@ actual model inference, not mocks. import os -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" - import pytest from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" @@ -59,8 +61,6 @@ tts_server_params = [ ] -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_001(omni_server, openai_client) -> None: @@ -92,8 +92,6 @@ def test_voice_001(omni_server, openai_client) -> None: raise -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_002(omni_server, openai_client) -> None: @@ -117,8 +115,6 @@ def test_voice_002(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_003(omni_server, openai_client) -> None: @@ -142,8 +138,6 @@ def test_voice_003(omni_server, openai_client) -> None: openai_client.send_audio_speech_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_language_001(omni_server, openai_client) -> None: diff --git a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py index c2461977d..bb8bf7dca 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py @@ -11,6 +11,8 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " @@ -108,8 +110,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"), @@ -138,8 +138,6 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"), diff --git a/tests/e2e/online_serving/test_qwen_image_expansion.py b/tests/e2e/online_serving/test_qwen_image_expansion.py index b6f91c13d..0830e4944 100644 --- a/tests/e2e/online_serving/test_qwen_image_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_expansion.py @@ -15,6 +15,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." NEGATIVE_PROMPT = "blurry, low quality" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) @@ -117,8 +119,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("Qwen/Qwen-Image"), @@ -142,8 +142,6 @@ def test_qwen_image(omni_server: OmniServer, openai_client: OpenAIClientHandler) openai_client.send_diffusion_request(request_config) -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases("Qwen/Qwen-Image-2512"), diff --git a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py index b958cfc05..4e79beab7 100644 --- a/tests/e2e/online_serving/test_qwen_image_layered_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_layered_expansion.py @@ -18,6 +18,8 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import decode_b64_image, generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + MODEL = "Qwen/Qwen-Image-Layered" EDIT_PROMPT = "Decompose this image into layers." EMPTY_EDIT_PROMPT = "" @@ -71,8 +73,6 @@ FEATURE_CASES = [ ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize("omni_server", FEATURE_CASES, indirect=True) def test_feature(omni_server: OmniServer, openai_client: OpenAIClientHandler): """Test feature combinations with Qwen-Image-Layered.""" @@ -149,8 +149,6 @@ def _collect_image_url_items(openai_client: OpenAIClientHandler, request_config: return image_items -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server, expected_layers", LAYERS_GUARD_CASES, @@ -224,8 +222,6 @@ PROMPT_CASES = [ ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize("omni_server", PROMPT_CASES, indirect=True) def test_empty_prompt(omni_server: OmniServer, openai_client: OpenAIClientHandler): """Test feature combinations with Qwen-Image-Layered.""" diff --git a/tests/e2e/online_serving/test_sd3_expansion.py b/tests/e2e/online_serving/test_sd3_expansion.py index 37e590ba3..09b50d2e5 100644 --- a/tests/e2e/online_serving/test_sd3_expansion.py +++ b/tests/e2e/online_serving/test_sd3_expansion.py @@ -7,6 +7,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4) POSITIVE_PROMPT = "A serene mountain landscape at sunset" NEGATIVE_PROMPT = "blurry, low quality, distorted" @@ -35,8 +37,6 @@ def _get_diffusion_feature_cases(model: str): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases( diff --git a/tests/e2e/online_serving/test_wan22_expansion.py b/tests/e2e/online_serving/test_wan22_expansion.py index 7e5bc9121..36035f649 100644 --- a/tests/e2e/online_serving/test_wan22_expansion.py +++ b/tests/e2e/online_serving/test_wan22_expansion.py @@ -23,6 +23,8 @@ from tests.helpers.mark import hardware_marks from tests.helpers.media import generate_synthetic_image from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + PROMPT = "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." NEGATIVE_PROMPT = "low quality, blurry, distorted face, extra limbs, bad anatomy, watermark, logo, text, ugly, deformed, mutated, jpeg artifacts" SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) @@ -79,8 +81,6 @@ def _get_wan22_feature_cases(): return cases -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_wan22_feature_cases(), diff --git a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py index 1f7a8c072..4f0e96447 100644 --- a/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py +++ b/tests/e2e/online_serving/test_wan_2_1_vace_expansion.py @@ -26,6 +26,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers" PROMPT = "A cat walking slowly across a sunlit garden path" @@ -131,8 +133,6 @@ def _get_vace_feature_cases(): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_vace_feature_cases(), diff --git a/tests/e2e/online_serving/test_zimage_expansion.py b/tests/e2e/online_serving/test_zimage_expansion.py index 679233c82..d730db0ca 100644 --- a/tests/e2e/online_serving/test_zimage_expansion.py +++ b/tests/e2e/online_serving/test_zimage_expansion.py @@ -16,6 +16,8 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + MODEL = "Tongyi-MAI/Z-Image-Turbo" PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard." @@ -101,8 +103,6 @@ def _get_diffusion_feature_cases(): ] -@pytest.mark.advanced_model -@pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", _get_diffusion_feature_cases(), diff --git a/tests/examples/offline_inference/test_text_to_image.py b/tests/examples/offline_inference/test_text_to_image.py index f24506587..041c32dc4 100644 --- a/tests/examples/offline_inference/test_text_to_image.py +++ b/tests/examples/offline_inference/test_text_to_image.py @@ -11,7 +11,8 @@ from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet from tests.helpers.assertions import assert_image_valid from tests.helpers.mark import hardware_marks -pytestmark = [pytest.mark.advanced_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] +pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] + T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py" README_PATH = T2I_SCRIPT.with_name("README.md") diff --git a/tests/examples/online_serving/test_qwen2_5_omni.py b/tests/examples/online_serving/test_qwen2_5_omni.py index b8acfef84..b3e49b8d9 100644 --- a/tests/examples/online_serving/test_qwen2_5_omni.py +++ b/tests/examples/online_serving/test_qwen2_5_omni.py @@ -4,9 +4,6 @@ See examples/online_serving/qwen2_5_omni/README.md """ import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - from pathlib import Path import pytest @@ -21,7 +18,9 @@ from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_te from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.advanced_model, pytest.mark.example] +pytestmark = [pytest.mark.full_model, pytest.mark.example, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" models = ["Qwen/Qwen2.5-Omni-7B"] @@ -40,8 +39,6 @@ test_params = [ common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: @@ -77,8 +74,6 @@ def test_send_multimodal_request_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: @@ -114,8 +109,6 @@ def test_send_multimodal_request_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: @@ -141,8 +134,6 @@ def test_send_multimodal_request_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: @@ -171,8 +162,6 @@ def test_modality_control_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: @@ -200,8 +189,6 @@ def test_modality_control_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: @@ -238,8 +225,6 @@ def test_modality_control_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2}) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: diff --git a/tests/examples/online_serving/test_qwen3_omni.py b/tests/examples/online_serving/test_qwen3_omni.py index e2c3b3384..e52a2bf5a 100644 --- a/tests/examples/online_serving/test_qwen3_omni.py +++ b/tests/examples/online_serving/test_qwen3_omni.py @@ -4,9 +4,6 @@ See examples/online_serving/qwen3_omni/README.md """ import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - from pathlib import Path import pytest @@ -21,7 +18,9 @@ from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_te from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -pytestmark = [pytest.mark.advanced_model, pytest.mark.example] +pytestmark = [pytest.mark.full_model, pytest.mark.example, pytest.mark.omni] + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] @@ -39,8 +38,6 @@ test_params = [ common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")] -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_001(omni_server) -> None: @@ -69,8 +66,6 @@ def test_send_multimodal_request_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_002(omni_server) -> None: @@ -102,8 +97,6 @@ def test_send_multimodal_request_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_send_multimodal_request_003(omni_server) -> None: @@ -119,8 +112,6 @@ def test_send_multimodal_request_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_001(omni_server) -> None: @@ -143,8 +134,6 @@ def test_modality_control_001(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_002(omni_server) -> None: @@ -167,8 +156,6 @@ def test_modality_control_002(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_modality_control_003(omni_server) -> None: @@ -199,8 +186,6 @@ def test_modality_control_003(omni_server) -> None: # TODO: Verify the E2E latency after confirmation baseline. -@pytest.mark.advanced_model -@pytest.mark.omni @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_stream_001(omni_server) -> None: diff --git a/tests/examples/online_serving/test_text_to_image.py b/tests/examples/online_serving/test_text_to_image.py index 6f89cb549..ee0a1fedb 100644 --- a/tests/examples/online_serving/test_text_to_image.py +++ b/tests/examples/online_serving/test_text_to_image.py @@ -18,7 +18,7 @@ from tests.helpers.assertions import assert_image_valid from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams -pytestmark = [pytest.mark.advanced_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] +pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] T2I_ONLINE_CLIENT = EXAMPLES / "online_serving" / "text_to_image" / "openai_chat_client.py" EXAMPLE_OUTPUT_SUBFOLDER = "example_online_t2i" diff --git a/tests/helpers/assertions.py b/tests/helpers/assertions.py index 533806477..604b76b62 100644 --- a/tests/helpers/assertions.py +++ b/tests/helpers/assertions.py @@ -11,7 +11,9 @@ import numpy as np import soundfile as sf from PIL import Image -from tests.helpers.media import cosine_similarity_text +from tests.helpers.media import ( + cosine_similarity_text, +) _GENDER_PIPELINE = None _GENDER_PIPELINE_LOCK = threading.Lock() @@ -56,7 +58,7 @@ def assert_image_diffusion_response( f"Expected {num_outputs_per_prompt} images, got {len(response.images)}" ) - if run_level == "advanced_model": + if run_level in {"advanced_model", "full_model"}: width = extra_body.get("width") height = extra_body.get("height") @@ -407,7 +409,8 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve modalities = request_config.get("modalities", ["text", "audio"]) - if run_level == "advanced_model": + if run_level in {"advanced_model", "full_model"}: + # Verify output success if "audio" in modalities: assert response.audio_content is not None, "No audio output is generated" print(f"audio content is: {response.audio_content}") @@ -417,12 +420,11 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve response.audio_bytes, speaker, ) - if "text" in modalities: assert response.text_content is not None, "No text output is generated" print(f"text content is: {response.text_content}") - # Verify image description + # Verify keywords in output word_types = ["text", "image", "audio", "video"] keywords_dict = request_config.get("key_words", {}) for word_type in word_types: @@ -441,11 +443,25 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve ) # Verify similarity (Whisper transcript vs streamed/detokenized text) - if "text" in modalities and "audio" in modalities: - assert response.similarity is not None and response.similarity > 0.9, ( - "The audio content is not same as the text" - ) - print(f"similarity is: {response.similarity}") + if "audio" in modalities: + audio_ref_text = request_config.get("audio_ref_text") + if "text" in modalities: + transcript = (response.audio_content or "").strip() + text_output = (response.text_content or "").strip() + similarity = cosine_similarity_text( + transcript.lower(), + text_output.lower(), + ) + assert similarity > 0.9, "The audio content is not same as the text" + print(f"similarity is: {similarity}") + if audio_ref_text: + audio_similarity = cosine_similarity_text( + response.audio_content.lower(), + str(audio_ref_text).lower(), + ) + assert audio_similarity > 0.9, ( + f"The audio content does not match reference text: similarity={audio_similarity:.3f}" + ) def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str) -> None: @@ -464,7 +480,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any], elif req_fmt == "wav" and response.audio_format: assert req_fmt in response.audio_format - if run_level == "advanced_model" and req_fmt != "pcm": + if run_level in {"advanced_model", "full_model"} and req_fmt != "pcm": expected_text = request_config.get("input") if expected_text: transcript = (response.audio_content or "").strip() diff --git a/tests/helpers/fixtures/run_args.py b/tests/helpers/fixtures/run_args.py index b18a64b98..975584d20 100644 --- a/tests/helpers/fixtures/run_args.py +++ b/tests/helpers/fixtures/run_args.py @@ -6,8 +6,8 @@ def pytest_addoption(parser): "--run-level", action="store", default="core_model", - choices=["core_model", "advanced_model"], - help="Test level to run: L2, L3", + choices=["core_model", "advanced_model", "full_model"], + help="Test level to run: L2, L3, L4", ) diff --git a/tests/helpers/fixtures/runtime.py b/tests/helpers/fixtures/runtime.py index 77a468f2f..4cae13cd6 100644 --- a/tests/helpers/fixtures/runtime.py +++ b/tests/helpers/fixtures/runtime.py @@ -36,7 +36,7 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st model = model_prefix + params.model port = params.port stage_config_path = params.stage_config_path - if run_level == "advanced_model" and stage_config_path is not None: + if run_level in {"advanced_model", "full_model"} and stage_config_path is not None: with open(stage_config_path, encoding="utf-8") as f: cfg = yaml.safe_load(f) or {} # Strip ``load_format: dummy`` (CI overlay default) so advanced_model diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 9520aff0c..190cb66b4 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -35,7 +35,6 @@ from tests.helpers.env import run_forced_gpu_cleanup_round from tests.helpers.media import ( _merge_base64_audio_to_segment, convert_audio_bytes_to_text, - cosine_similarity_text, decode_b64_image, ) from vllm_omni.config.stage_config import resolve_deploy_yaml @@ -500,7 +499,6 @@ class OmniResponse: audio_content: str | None = None audio_format: str | None = None audio_bytes: bytes | None = None - similarity: float | None = None e2e_latency: float | None = None success: bool = False error_message: str | None = None @@ -542,19 +540,15 @@ class OpenAIClientHandler: text_content += content result.e2e_latency = time.perf_counter() - start_time audio_content = None - similarity = None if audio_data: merged_seg = _merge_base64_audio_to_segment(audio_data) wav_buf = BytesIO() merged_seg.export(wav_buf, format="wav") result.audio_bytes = wav_buf.getvalue() audio_content = convert_audio_bytes_to_text(result.audio_bytes) - if audio_content and text_content: - similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) result.text_content = text_content result.audio_data = audio_data result.audio_content = audio_content - result.similarity = similarity result.success = True except Exception as e: result.error_message = f"Stream processing error: {str(e)}" @@ -578,15 +572,11 @@ class OpenAIClientHandler: result.cached_tokens = details.cached_tokens result.e2e_latency = time.perf_counter() - start_time audio_content = None - similarity = None if audio_data: result.audio_bytes = base64.b64decode(audio_data) audio_content = convert_audio_bytes_to_text(result.audio_bytes) - if audio_content and text_content: - similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) result.text_content = text_content result.audio_content = audio_content - result.similarity = similarity result.success = True except Exception as e: result.error_message = f"Non-stream processing error: {str(e)}"