[CI] Update test markers and configurations to use 'full_model' for L4 nightly tests (#2641)

Signed-off-by: wangyu <410167048@qq.com>

.buildkite/test-nightly.yml

@@ -13,7 +13,7 @@ steps:
       - label: ":full_moon: Omni · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and H100 and omni" --run-level "advanced_model"
+          - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -52,7 +52,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "advanced_model and omni and L4" --run-level "advanced_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
         agents:
           queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -70,7 +70,7 @@ steps:
       - label: ":full_moon: Omni · Doc Test with H100"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/examples/ -m "advanced_model and omni and H100" --run-level "advanced_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -160,7 +160,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and L4 and omni" --run-level "advanced_model"
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -233,7 +233,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100"
         timeout_in_minutes: 120
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion and not test_wan_2_1_vace_expansion and not hunyuan" -m "advanced_model and diffusion and H100" --run-level "advanced_model"
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -271,7 +271,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion and not test_wan_2_1_vace_expansion and not hunyuan" -m "advanced_model and diffusion and L4" --run-level "advanced_model"
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -290,7 +290,7 @@ steps:
         timeout_in_minutes: 60
         commands:
           - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-          - pytest -s -v tests/examples/*/test_text_to_image.py -m "advanced_model and example and H100" --run-level "advanced_model"
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -328,7 +328,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test"
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level advanced_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
+          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
           - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
         agents:
           queue: "mithril-h100-pool"
@@ -367,7 +367,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test"
         timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level advanced_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
+          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
           - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
           - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
         agents:
@@ -409,7 +409,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level advanced_model
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -514,7 +514,7 @@ steps:
       - label: ":full_moon: Diffusion X2V · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "advanced_model" --run-level "advanced_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -552,7 +552,7 @@ steps:
       - label: ":full_moon: Diffusion X2V · Accuracy Test"
         timeout_in_minutes: 180
         commands:
-          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m advanced_model --run-level advanced_model
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:

.buildkite/test-ready.yml

@@ -120,23 +120,6 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "Audio Generation Model Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: "Diffusion Cache Backend Test"
     depends_on: upload-ready-pipeline
     commands:

benchmarks/accuracy/README.md

@@ -23,5 +23,5 @@ Test guidance:
 - Local static/self-checks live in `tests/benchmarks/test_accuracy_bench_utils.py`.
 - End-to-end generation/evaluation should be validated in a remote GPU
   environment. In the current repo marker system there is `L4` but no `L5`
-  marker, so benchmark smoke tests should be wired as `advanced_model +
-  benchmark + L4` when GPU capacity is available.
+  marker, so benchmark smoke tests should be wired as `full_model +
+  benchmark + L4` for nightly when GPU capacity is available.

benchmarks/accuracy/image_to_image/README.md

@@ -99,5 +99,5 @@ Notes:
 - This flow requires the optional Hugging Face `datasets` package.
 - `generate` writes `generation_manifest.json` with local output coverage.
 - The current repo marker set exposes `L4` but not `L5`, so if you promote an
-  end-to-end smoke test into CI, use the existing `advanced_model`, `benchmark`,
-  and `L4` markers or introduce a new repo-wide marker explicitly first.
+  end-to-end smoke test into CI, use the `full_model`, `benchmark`,
+  and `L4` markers for nightly (or `advanced_model` for merge) or introduce a new repo-wide marker explicitly first.

docs/contributing/ci/CI_5levels.md

@@ -418,13 +418,13 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
     **Explanation**:
 
-    @pytest.mark.advanced_model: Marks the test as L3 or L4 level, indicating that this test case performs deep validation, using real models for performance, integration, and accuracy testing. This forms a "basic-advanced" correspondence with the core_model mark at the L2 level.
+    @pytest.mark.advanced_model: Marks the test as L3 merge level, indicating deep validation with real models. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples).
 
     @pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly.
 
     @pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing.
 
-    **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation run logic at the merge or nightly level, you can mark it only with @pytest.mark.advanced_model. If you believe the test case needs to accommodate both basic run and deep validation test logic, you should mark it with both @pytest.mark.core_model and @pytest.mark.advanced_model.
+    **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation at merge (L3), use @pytest.mark.advanced_model. For L4 nightly-only expansion and doc-example tests, use @pytest.mark.full_model with `--run-level full_model`. If the test case needs both basic run and deep validation, mark with @pytest.mark.core_model and the appropriate L3/L4 marker (`advanced_model` and/or `full_model`).
 
     **2.4.2 Test Function Definition and Documentation**
 
@@ -516,9 +516,11 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
     **Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n.
 
-    **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model performs deep validation.
+    **Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model and full_model perform deep validation.
 
--   ***Run Command***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
+-   ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
+
+-   ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model`
 
 ## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing
 

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -37,7 +37,7 @@ Currently all the features are available in online serving mode. Hence, only nee
 **Code Style**
 
 - Validation: test that the multimodal output files of your model have the correct shapes. `OpenAIClientHandler.send_diffusion_request` should have taken care of this.
-- Test marks: always add `advanced_model` and `diffusion`. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/).
+- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/).
 - To maximize code reuse, you may refer to
     - `tests/conftest.py` for `omni_server` (running server in subprocess) and `openai_client` fixtures (sending requests and validating output), `generate_synthetic_image` and `assert_XXX_valid` helper.
     - `tests/helpers/mark.py` for `@hardware_test(...)` and `hardware_marks`.

docs/contributing/ci/test_guide.md

@@ -42,31 +42,46 @@ Our test scripts use the pytest framework. First, please use `git clone https://
     ```
     The latest test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml).
 
-=== "L3 level & L4 level"
+=== "L3 level"
 
     ```bash
     pytest -s -v -m "advanced_model" --run-level=advanced_model
     ```
-    If you only want to run L3 test case, you can use:
-    ```bash
-    pytest -s -v e2e/ --ignore-glob='*expansion.py' -m "advanced_model" --run-level=advanced_model
-    ```
     If you only want to run a specific test case, you can use:
     ```bash
     pytest -s -v test_xxxx.py --run-level=advanced_model
     ```
     If you only want to run specific test cases on a particular platform, you can use:
     ```bash
-    pytest -s -v -m "core_model and distributed_cuda and L4"  --run-level=core_model
+    pytest -s -v -m "advanced_model and distributed_cuda and L4"  --run-level=advanced_model
+    ```
+    The latest L3 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml).
+
+
+=== "L4 level"
+
+    ```bash
+    cd tests
+    pytest -s -v -m "full_model" --run-level=full_model
+    ```
+    If you only want to run a specific test case, you can use:
+    ```bash
+    pytest -s -v test_xxxx.py --run-level=full_model
+    ```
+    If you only want to run specific test cases on a particular platform, you can use:
+    ```bash
+    pytest -s -v -m "full_model and distributed_cuda and L4"  --run-level=full_model
     ```
     Note: To run performance tests (defaults to ``test_qwen_omni.json``; use ``--test-config-file tests/dfx/perf/tests/test_tts.json`` for TTS):
     ```bash
     pytest -s -v tests/dfx/perf/scripts/run_benchmark.py
     ```
+    The latest L4 (nightly) test commands use the `full_model` marker and `--run-level full_model` (see [test-nightly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml) and [test-nightly-diffusion.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly-diffusion.yml)). Example:
 
-    The latest L3 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-merge.yml).
-
-    The latest L4 test commands for various test suites can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml).
+    ```bash
+    cd tests
+    pytest -s -v -m "full_model and omni and H100" --run-level=full_model
+    ```
 
 You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
 

docs/contributing/ci/tests_markers.md

@@ -8,7 +8,8 @@ Defined in `pyproject.toml`:
 | Marker             | Description                                               |
 | ------------------ | --------------------------------------------------------- |
 | `core_model`       | L1&L2 tests (run in each PR)                              |
-| `advanced_model`   | L3&L4 level tests (run in each merge or nightly)          |
+| `advanced_model`   | L3 tests (run on each merge to main)                 |
+| `full_model`       | L4 tests (run nightly) |
 | `diffusion`        | Diffusion model tests                                     |
 | `omni`             | Omni model tests                                          |
 | `cache`            | Cache backend tests                                       |

pyproject.toml

@@ -179,7 +179,8 @@ addopts = [
 markers = [
     # ci/cd required
     "core_model: L1&L2 tests (run in each PR)",
-    "advanced_model: L3&L4 level tests (run in each merge or nightly)",
+    "advanced_model: L3 level tests (run on each merge)",
+    "full_model: L4 level tests (run nightly)",
     # function module markers
     "diffusion: Diffusion model tests",
     "omni: Omni model tests",

tests/benchmarks/test_accuracy_bench_utils.py

@@ -6,6 +6,9 @@ from pathlib import Path
 import pytest
 from PIL import Image
 
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+
+
 REPO_ROOT = Path(__file__).resolve().parents[2]
 if str(REPO_ROOT) not in sys.path:
     sys.path.insert(0, str(REPO_ROOT))
@@ -37,8 +40,6 @@ from benchmarks.accuracy.text_to_image.gbench import (
     summarize_gebench_results,
 )
 
-pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
-
 
 def test_summarize_gebench_generated_records_groups_by_type():
     records = [

tests/dfx/perf/scripts/run_benchmark.py

@@ -17,6 +17,9 @@ from tests.dfx.conftest import (
 )
 from tests.helpers.runtime import OmniServer
 
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
+
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
@@ -293,6 +296,7 @@ def assert_result(
                 print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
 def test_performance_benchmark(omni_server, benchmark_params):

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -32,6 +32,8 @@ from typing import Any, cast
 import psutil
 import pytest
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
@@ -662,8 +664,7 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
 # ---------------------------------------------------------------------------
 # Test entry point
 # ---------------------------------------------------------------------------
-
-
+@pytest.mark.benchmark
 @pytest.mark.parametrize(
     "diffusion_server",
     server_params,

tests/diffusion/models/glm_image/test_glm_image_sp.py

@@ -6,6 +6,8 @@ import pytest
 
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
 
 @pytest.fixture(scope="function", autouse=True)
 def setup_sp_groups(mocker):
@@ -24,9 +26,6 @@ def setup_sp_groups(mocker):
     yield
 
 
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-
 def test_glm_image_sp_plan_defined():
     """Test that _sp_plan is properly defined on GlmImageTransformer2DModel."""
     from vllm_omni.diffusion.models.glm_image.glm_image_transformer import (

tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py

@@ -17,8 +17,6 @@ from vllm_omni.distributed.omni_coordinator import (
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
 
 def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]:
     ctx = zmq.Context.instance()

tests/e2e/accuracy/test_gebench_h100_smoke.py

@@ -9,10 +9,10 @@ from benchmarks.accuracy.text_to_image.gbench import main as gbench_main
 from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
 from tests.helpers.mark import hardware_test
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_gebench_h100_smoke(
     gebench_accuracy_servers,

tests/e2e/accuracy/test_gedit_bench_h100_smoke.py

@@ -10,10 +10,10 @@ from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main
 from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
 from tests.helpers.mark import hardware_test
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_gedit_bench_h100_smoke(
     gedit_accuracy_servers,

tests/e2e/accuracy/test_qwen_image.py

@@ -17,6 +17,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServer
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
+
 MODEL_ID = "Qwen/Qwen-Image"
 MODEL_ENV_VAR = "QWEN_IMAGE_MODEL"
 PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
@@ -99,9 +102,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
         run_post_test_cleanup(enable_force=True)
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:
     model = _model_name()

tests/e2e/accuracy/test_qwen_image_edit.py

@@ -15,6 +15,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServer
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
+
 SINGLE_MODEL = "Qwen/Qwen-Image-Edit"
 MULTIPLE_MODEL = "Qwen/Qwen-Image-Edit-2509"
 WIDTH = 512
@@ -169,9 +172,7 @@ def _diffusers_output_multiple_image(
     )
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_qwen_image_edit_single_matches_diffusers(
     accuracy_artifact_root: Path,
@@ -196,9 +197,7 @@ def test_qwen_image_edit_single_matches_diffusers(
     )
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 @pytest.mark.skip(
     reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged."

tests/e2e/accuracy/test_qwen_image_layered.py

@@ -17,6 +17,9 @@ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServer
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
+
 MODEL_ID = "Qwen/Qwen-Image-Layered"
 MODEL_ENV_VAR = "QWEN_IMAGE_LAYERED_MODEL"
 PROMPT = "decompose into layers"
@@ -126,9 +129,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
         run_post_test_cleanup(enable_force=True)
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_qwen_image_layered_matches_diffusers(accuracy_artifact_root: Path, qwen_bear_image: Image.Image) -> None:
     model = _model_name()

tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py

@@ -50,6 +50,8 @@ from tests.e2e.accuracy.wan22_i2v.wan22_i2v_video_similarity_common import (
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServerParams
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 
 def test_parse_video_metadata_extracts_dimensions_and_fps() -> None:
     payload = {
@@ -537,9 +539,7 @@ def _generate_offline_video(*, image_source: str) -> tuple[Path, Path]:
     return offline_path, offline_metadata_path
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=1)
 def test_wan22_i2v_diffusers_offline_generates_video(
     wan22_i2v_image_source: str | None,
@@ -563,9 +563,7 @@ def test_wan22_i2v_diffusers_offline_generates_video(
     assert offline_metadata["frame_count"] == NUM_FRAMES
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
 def test_wan22_i2v_online_serving_generates_video(
@@ -594,9 +592,7 @@ def test_wan22_i2v_online_serving_generates_video(
     assert online_metadata["frame_count"] == NUM_FRAMES
 
 
-@pytest.mark.advanced_model
 @pytest.mark.benchmark
-@pytest.mark.diffusion
 @hardware_test(res={"cuda": "H100"}, num_cards=2)
 def test_wan22_i2v_serving_matches_diffusers_video_similarity(
     wan22_i2v_image_source: str | None,

tests/e2e/offline_inference/test_quantization_fp8.py

@@ -27,9 +27,6 @@ Usage:
 """
 
 import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
 from pathlib import Path
 from typing import Any
 
@@ -44,6 +41,8 @@ from vllm_omni.platforms import current_omni_platform
 
 pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
 
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
 # ─── helpers ──────────────────────────────────────────────────────────────────
 
 

tests/e2e/offline_inference/test_stable_audio_expansion.py

@@ -22,6 +22,8 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
 _SAMPLE_RATE = 44100
 _CLIP_DURATION_S = 2.0
 
@@ -72,8 +74,6 @@ def generate_stable_audio_short_clip(
     return audio
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.cache
 @hardware_test(res={"cuda": "L4", "xpu": "B60"})
 def test_stable_audio_quantization_and_teacache() -> None:

tests/e2e/online_serving/test_bagel_expansion.py

@@ -19,6 +19,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution."
 NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark"
 
@@ -112,8 +114,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("ByteDance-Seed/BAGEL-7B-MoT"),

tests/e2e/online_serving/test_dynin_omni_expansion.py

@@ -5,29 +5,26 @@ Example online tests for Dynin-Omni model.
 """
 
 import base64
-import gc
 import os
 from io import BytesIO
 from pathlib import Path
 
-import numpy as np
 import pytest
-import soundfile as sf
 from vllm.assets.image import ImageAsset
 
 from tests.helpers.mark import hardware_test
-from tests.helpers.media import convert_audio_bytes_to_text
 from tests.helpers.runtime import OmniServerParams
 
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
 MODEL = "snu-aidas/Dynin-Omni"
 STAGE_CONFIG = str(Path(__file__).parent.parent / "stage_configs" / "dynin_omni_ci.yaml")
-_WHISPER_SAMPLE_RATE_HZ = 16_000
 
 T2I_PROMPT = "A high quality detailed living room interior photo."
-T2S_PROMPT = "Please read this sentence naturally: Hello from Dynin-Omni online serving."
+T2S_PROMPT = "Please read this sentence naturally: Hello from online serving."
 I2I_PROMPT = "Transform this outdoor nature boardwalk scene into a painting style with vivid colors."
 
 TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, stage_init_timeout=600)]
@@ -35,65 +32,6 @@ _STAGE_COUNT = 3
 _I2I_STAGE_SAMPLING = {"max_tokens": 1, "temperature": 0.0, "top_p": 1.0, "detokenize": False}
 
 
-def _prepare_audio_waveform_for_whisper(audio_data: np.ndarray, samplerate: int) -> np.ndarray:
-    """Normalize decoded audio into a mono 16 kHz float32 waveform for Whisper."""
-    if samplerate <= 0:
-        raise ValueError(f"Invalid audio sample rate: {samplerate}")
-
-    waveform = np.asarray(audio_data, dtype=np.float32)
-    if waveform.ndim == 0:
-        raise ValueError("Audio waveform must have at least one dimension")
-    if waveform.ndim > 1:
-        waveform = np.mean(waveform, axis=1)
-    if waveform.size == 0:
-        raise ValueError("Empty audio waveform")
-
-    if samplerate != _WHISPER_SAMPLE_RATE_HZ:
-        target_num_samples = max(int(round(waveform.shape[0] * _WHISPER_SAMPLE_RATE_HZ / samplerate)), 1)
-        source_positions = np.arange(waveform.shape[0], dtype=np.float64)
-        target_positions = np.linspace(
-            0.0,
-            max(waveform.shape[0] - 1, 0),
-            num=target_num_samples,
-            dtype=np.float64,
-        )
-        waveform = np.interp(target_positions, source_positions, waveform).astype(np.float32)
-
-    return np.ascontiguousarray(np.clip(waveform, -1.0, 1.0), dtype=np.float32)
-
-
-def _convert_audio_bytes_to_text_without_ffmpeg(raw_bytes: bytes) -> str:
-    """Dynin t2s keeps Whisper transcription local to this test module and avoids ffmpeg."""
-    import whisper
-
-    data, samplerate = sf.read(BytesIO(raw_bytes), dtype="float32", always_2d=True)
-    audio_waveform = _prepare_audio_waveform_for_whisper(data, samplerate)
-
-    model = whisper.load_model("small", device="cpu")
-    try:
-        transcript = model.transcribe(
-            audio_waveform,
-            temperature=0.0,
-            word_timestamps=True,
-            condition_on_previous_text=False,
-        )["text"]
-    finally:
-        del model
-        gc.collect()
-
-    return transcript or ""
-
-
-@pytest.fixture
-def dynin_t2s_openai_client(openai_client, monkeypatch):
-    monkeypatch.setattr(
-        convert_audio_bytes_to_text,
-        "convert_audio_bytes_to_text",
-        _convert_audio_bytes_to_text_without_ffmpeg,
-    )
-    return openai_client
-
-
 def _build_t2i_messages(prompt: str) -> list[dict]:
     return [{"role": "user", "content": [{"type": "text", "text": f"<|t2i|> {prompt}"}]}]
 
@@ -118,8 +56,6 @@ def _build_i2i_messages(prompt: str) -> list[dict]:
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"})
 @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
 def test_send_i2i_request_001(omni_server, openai_client) -> None:
@@ -134,8 +70,6 @@ def test_send_i2i_request_001(omni_server, openai_client) -> None:
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"})
 @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
 def test_send_t2i_request_001(omni_server, openai_client) -> None:
@@ -147,14 +81,13 @@ def test_send_t2i_request_001(omni_server, openai_client) -> None:
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.core_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"})
 @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
-def test_send_t2s_request_001(omni_server, dynin_t2s_openai_client) -> None:
+def test_send_t2s_request_001(omni_server, openai_client) -> None:
     request_config = {
         "model": omni_server.model,
         "messages": _build_t2s_messages(T2S_PROMPT),
         "modalities": ["audio"],
+        "audio_ref_text": T2S_PROMPT,
     }
-    dynin_t2s_openai_client.send_omni_request(request_config)
+    openai_client.send_omni_request(request_config)

tests/e2e/online_serving/test_flux2_expansion.py

@@ -14,6 +14,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
 POSITIVE_PROMPT = "A cat sitting on a windowsill"
 NEGATIVE_PROMPT = "blurry, low quality"
@@ -86,8 +88,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(

tests/e2e/online_serving/test_flux2_klein_inpaint_expansion.py

@@ -16,6 +16,8 @@ from PIL import Image, ImageDraw
 
 from tests.helpers.runtime import OmniServer, OmniServerParams
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
 MODEL = "black-forest-labs/FLUX.2-klein-4B"
 
 _HEIGHT = 512
@@ -66,8 +68,6 @@ def _send_edit_request(host: str, port: int, model: str, image_b64: str, mask_b6
         return response.json()
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
 def test_flux2_klein_inpaint_basic(omni_server: OmniServer):
     input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128)))
@@ -92,8 +92,6 @@ def test_flux2_klein_inpaint_basic(omni_server: OmniServer):
     assert img.size == (_WIDTH, _HEIGHT)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
 def test_flux2_klein_inpaint_deterministic(omni_server: OmniServer):
     input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128)))
@@ -137,8 +135,6 @@ def test_flux2_klein_inpaint_deterministic(omni_server: OmniServer):
     )
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
 def test_flux2_klein_inpaint_multiple_outputs(omni_server: OmniServer):
     input_image_b64 = _image_to_base64_jpeg(Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128)))

tests/e2e/online_serving/test_flux_2_dev_expansion.py

@@ -17,6 +17,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 MODEL = "black-forest-labs/FLUX.2-dev"
 PROMPT = "A cinematic mountain landscape at sunrise, dramatic clouds, ultra-detailed, realistic photography."
 NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark"
@@ -54,8 +56,6 @@ def _get_flux_2_dev_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_flux_2_dev_feature_cases(MODEL),

tests/e2e/online_serving/test_flux_kontext_expansion.py

@@ -8,6 +8,8 @@ import pytest
 from tests.helpers.media import generate_synthetic_image
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
 NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
 MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
@@ -29,8 +31,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),
@@ -54,8 +54,6 @@ def test_flux_kontext_text_to_image(omni_server: OmniServer, openai_client: Open
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),
@@ -83,8 +81,6 @@ def test_flux_kontext_image_edit(omni_server: OmniServer, openai_client: OpenAIC
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),
@@ -110,8 +106,6 @@ def test_flux_kontext_image_edit_no_negative(omni_server: OmniServer, openai_cli
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),
@@ -135,8 +129,6 @@ def test_flux_kontext_high_resolution(omni_server: OmniServer, openai_client: Op
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),

tests/e2e/online_serving/test_hunyuan_video_15_expansion.py

@@ -14,6 +14,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion."
 NEGATIVE_PROMPT = "low quality, blurry, distorted"
 
@@ -62,8 +64,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(MODEL),

tests/e2e/online_serving/test_longcat_image_edit_expansion.py

@@ -17,6 +17,8 @@ from tests.helpers.mark import hardware_marks
 from tests.helpers.media import generate_synthetic_image
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting."
 NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
 SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
@@ -50,8 +52,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"),

tests/e2e/online_serving/test_longcat_image_expansion.py

@@ -16,6 +16,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 TEXT_TO_IMAGE_PROMPT = (
     "A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed."
 )
@@ -51,8 +53,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("meituan-longcat/LongCat-Image"),

tests/e2e/online_serving/test_nextstep_expansion.py

@@ -14,6 +14,8 @@ from tests.helpers.runtime import (
     dummy_messages_from_mix_data,
 )
 
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
+
 # L4: 4 GPUs + TP=4; XPU B60: 2 cards (use num_cards={"cuda": 4, "xpu": 4} if needed)
 FOUR_CARD_MARKS = hardware_marks(
     res={"cuda": "L4", "xpu": "B60"},
@@ -45,8 +47,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(model=os.environ.get("VLLM_TEST_NEXTSTEP_MODEL", _DEFAULT_MODEL)),

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -6,8 +6,6 @@ E2E Online tests for Qwen3-Omni model.
 
 import os
 
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
 import pytest
 
 from tests.helpers.mark import hardware_test
@@ -15,6 +13,10 @@ from tests.helpers.media import generate_synthetic_audio, generate_synthetic_ima
 from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
 from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
 
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
 model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
 
 AUDIO_KEY = ["test"]
@@ -130,8 +132,6 @@ def get_max_batch_size(size_type="few"):
     return batch_sizes.get(size_type, 5)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_text_to_audio_001(omni_server, openai_client) -> None:
@@ -154,8 +154,6 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_text_to_text_audio_001(omni_server, openai_client) -> None:
@@ -176,8 +174,6 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_text_video_to_text_001(omni_server, openai_client) -> None:
@@ -202,8 +198,6 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None:
@@ -227,8 +221,6 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_text_audio_to_text_audio_002(omni_server, openai_client) -> None:
@@ -254,8 +246,6 @@ def test_text_audio_to_text_audio_002(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
@@ -280,8 +270,6 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_large_image_to_text_audio_001(omni_server, openai_client) -> None:
@@ -310,8 +298,6 @@ def test_large_image_to_text_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
@@ -338,8 +324,6 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
 
 
 @pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True)
 def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
@@ -369,8 +353,6 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.skip(reason="issue: #2827")
@@ -398,8 +380,6 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_audio_in_video_002(omni_server, openai_client) -> None:
@@ -427,8 +407,6 @@ def test_audio_in_video_002(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_one_word_prompt_001(omni_server, openai_client) -> None:
@@ -463,8 +441,6 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
             print(f"Similarity assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}")
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_speaker_001(omni_server, openai_client) -> None:
@@ -490,8 +466,6 @@ def test_speaker_001(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.skip(reason="Known issue: occasional inaccuracy in voice recognition.")
@@ -528,8 +502,6 @@ def test_speaker_002(omni_server, openai_client) -> None:
             print(f"Gender assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}")
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_speaker_003(omni_server, openai_client) -> None:
@@ -555,8 +527,6 @@ def test_speaker_003(omni_server, openai_client) -> None:
     openai_client.send_omni_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_language_001(omni_server, openai_client) -> None:

tests/e2e/online_serving/test_qwen3_tts_base_expansion.py

@@ -9,15 +9,17 @@ actual model inference, not mocks.
 
 import os
 
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
 import pytest
 
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import get_deploy_config_path
 
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
 MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
 
 REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
@@ -61,9 +63,6 @@ tts_server_params = [
 ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.core_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
@@ -89,9 +88,6 @@ def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
     openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size("few"))
 
 
-@pytest.mark.advanced_model
-@pytest.mark.core_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_response_format_001(omni_server, openai_client) -> None:

tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py

@@ -9,15 +9,17 @@ actual model inference, not mocks.
 
 import os
 
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
 import pytest
 
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import get_deploy_config_path
 
+pytestmark = [pytest.mark.full_model, pytest.mark.omni]
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
 MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
 
 
@@ -59,8 +61,6 @@ tts_server_params = [
 ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_voice_001(omni_server, openai_client) -> None:
@@ -92,8 +92,6 @@ def test_voice_001(omni_server, openai_client) -> None:
             raise
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_voice_002(omni_server, openai_client) -> None:
@@ -117,8 +115,6 @@ def test_voice_002(omni_server, openai_client) -> None:
     openai_client.send_audio_speech_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_voice_003(omni_server, openai_client) -> None:
@@ -142,8 +138,6 @@ def test_voice_003(omni_server, openai_client) -> None:
     openai_client.send_audio_speech_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
 def test_language_001(omni_server, openai_client) -> None:

tests/e2e/online_serving/test_qwen_image_edit_expansion.py

@@ -11,6 +11,8 @@ from tests.helpers.mark import hardware_marks
 from tests.helpers.media import generate_synthetic_image
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
 MULTI_EDIT_PROMPT = (
     "Transform the first image into a Dadaism collage art. "
@@ -108,8 +110,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
@@ -138,8 +138,6 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),

tests/e2e/online_serving/test_qwen_image_expansion.py

@@ -15,6 +15,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
 NEGATIVE_PROMPT = "blurry, low quality"
 SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
@@ -117,8 +119,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("Qwen/Qwen-Image"),
@@ -142,8 +142,6 @@ def test_qwen_image(omni_server: OmniServer, openai_client: OpenAIClientHandler)
     openai_client.send_diffusion_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases("Qwen/Qwen-Image-2512"),

tests/e2e/online_serving/test_qwen_image_layered_expansion.py

@@ -18,6 +18,8 @@ from tests.helpers.mark import hardware_marks
 from tests.helpers.media import decode_b64_image, generate_synthetic_image
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 MODEL = "Qwen/Qwen-Image-Layered"
 EDIT_PROMPT = "Decompose this image into layers."
 EMPTY_EDIT_PROMPT = ""
@@ -71,8 +73,6 @@ FEATURE_CASES = [
 ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize("omni_server", FEATURE_CASES, indirect=True)
 def test_feature(omni_server: OmniServer, openai_client: OpenAIClientHandler):
     """Test feature combinations with Qwen-Image-Layered."""
@@ -149,8 +149,6 @@ def _collect_image_url_items(openai_client: OpenAIClientHandler, request_config:
     return image_items
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server, expected_layers",
     LAYERS_GUARD_CASES,
@@ -224,8 +222,6 @@ PROMPT_CASES = [
 ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize("omni_server", PROMPT_CASES, indirect=True)
 def test_empty_prompt(omni_server: OmniServer, openai_client: OpenAIClientHandler):
     """Test feature combinations with Qwen-Image-Layered."""

tests/e2e/online_serving/test_sd3_expansion.py

@@ -7,6 +7,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
 POSITIVE_PROMPT = "A serene mountain landscape at sunset"
 NEGATIVE_PROMPT = "blurry, low quality, distorted"
@@ -35,8 +37,6 @@ def _get_diffusion_feature_cases(model: str):
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(

tests/e2e/online_serving/test_wan22_expansion.py

@@ -23,6 +23,8 @@ from tests.helpers.mark import hardware_marks
 from tests.helpers.media import generate_synthetic_image
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 PROMPT = "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage."
 NEGATIVE_PROMPT = "low quality, blurry, distorted face, extra limbs, bad anatomy, watermark, logo, text, ugly, deformed, mutated, jpeg artifacts"
 SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
@@ -79,8 +81,6 @@ def _get_wan22_feature_cases():
     return cases
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_wan22_feature_cases(),

tests/e2e/online_serving/test_wan_2_1_vace_expansion.py

@@ -26,6 +26,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 MODEL = "Wan-AI/Wan2.1-VACE-1.3B-diffusers"
 PROMPT = "A cat walking slowly across a sunlit garden path"
 
@@ -131,8 +133,6 @@ def _get_vace_feature_cases():
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_vace_feature_cases(),

tests/e2e/online_serving/test_zimage_expansion.py

@@ -16,6 +16,8 @@ import pytest
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
 
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
 MODEL = "Tongyi-MAI/Z-Image-Turbo"
 PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
 
@@ -101,8 +103,6 @@ def _get_diffusion_feature_cases():
     ]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
 @pytest.mark.parametrize(
     "omni_server",
     _get_diffusion_feature_cases(),

tests/examples/offline_inference/test_text_to_image.py

@@ -11,7 +11,8 @@ from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet
 from tests.helpers.assertions import assert_image_valid
 from tests.helpers.mark import hardware_marks
 
-pytestmark = [pytest.mark.advanced_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
+pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
+
 
 T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py"
 README_PATH = T2I_SCRIPT.with_name("README.md")

tests/examples/online_serving/test_qwen2_5_omni.py

@@ -4,9 +4,6 @@ See examples/online_serving/qwen2_5_omni/README.md
 """
 
 import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
 from pathlib import Path
 
 import pytest
@@ -21,7 +18,9 @@ from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_te
 from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import get_deploy_config_path
 
-pytestmark = [pytest.mark.advanced_model, pytest.mark.example]
+pytestmark = [pytest.mark.full_model, pytest.mark.example, pytest.mark.omni]
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
 models = ["Qwen/Qwen2.5-Omni-7B"]
 
@@ -40,8 +39,6 @@ test_params = [
 common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_001(omni_server) -> None:
@@ -77,8 +74,6 @@ def test_send_multimodal_request_001(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_002(omni_server) -> None:
@@ -114,8 +109,6 @@ def test_send_multimodal_request_002(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_003(omni_server) -> None:
@@ -141,8 +134,6 @@ def test_send_multimodal_request_003(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_001(omni_server) -> None:
@@ -171,8 +162,6 @@ def test_modality_control_001(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_002(omni_server) -> None:
@@ -200,8 +189,6 @@ def test_modality_control_002(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_003(omni_server) -> None:
@@ -238,8 +225,6 @@ def test_modality_control_003(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_stream_001(omni_server) -> None:

tests/examples/online_serving/test_qwen3_omni.py

@@ -4,9 +4,6 @@ See examples/online_serving/qwen3_omni/README.md
 """
 
 import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
 from pathlib import Path
 
 import pytest
@@ -21,7 +18,9 @@ from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_te
 from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import get_deploy_config_path
 
-pytestmark = [pytest.mark.advanced_model, pytest.mark.example]
+pytestmark = [pytest.mark.full_model, pytest.mark.example, pytest.mark.omni]
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
 models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
 
@@ -39,8 +38,6 @@ test_params = [
 common_args = ["python", os.path.join(example_dir, "openai_chat_completion_client_for_multimodal_generation.py")]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_001(omni_server) -> None:
@@ -69,8 +66,6 @@ def test_send_multimodal_request_001(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_002(omni_server) -> None:
@@ -102,8 +97,6 @@ def test_send_multimodal_request_002(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_send_multimodal_request_003(omni_server) -> None:
@@ -119,8 +112,6 @@ def test_send_multimodal_request_003(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_001(omni_server) -> None:
@@ -143,8 +134,6 @@ def test_modality_control_001(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_002(omni_server) -> None:
@@ -167,8 +156,6 @@ def test_modality_control_002(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_modality_control_003(omni_server) -> None:
@@ -199,8 +186,6 @@ def test_modality_control_003(omni_server) -> None:
     # TODO: Verify the E2E latency after confirmation baseline.
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_stream_001(omni_server) -> None:

tests/examples/online_serving/test_text_to_image.py

@@ -18,7 +18,7 @@ from tests.helpers.assertions import assert_image_valid
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams
 
-pytestmark = [pytest.mark.advanced_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
+pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
 
 T2I_ONLINE_CLIENT = EXAMPLES / "online_serving" / "text_to_image" / "openai_chat_client.py"
 EXAMPLE_OUTPUT_SUBFOLDER = "example_online_t2i"

tests/helpers/assertions.py

@@ -11,7 +11,9 @@ import numpy as np
 import soundfile as sf
 from PIL import Image
 
-from tests.helpers.media import cosine_similarity_text
+from tests.helpers.media import (
+    cosine_similarity_text,
+)
 
 _GENDER_PIPELINE = None
 _GENDER_PIPELINE_LOCK = threading.Lock()
@@ -56,7 +58,7 @@ def assert_image_diffusion_response(
             f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
         )
 
-    if run_level == "advanced_model":
+    if run_level in {"advanced_model", "full_model"}:
         width = extra_body.get("width")
         height = extra_body.get("height")
 
@@ -407,7 +409,8 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
 
     modalities = request_config.get("modalities", ["text", "audio"])
 
-    if run_level == "advanced_model":
+    if run_level in {"advanced_model", "full_model"}:
+        # Verify output success
         if "audio" in modalities:
             assert response.audio_content is not None, "No audio output is generated"
             print(f"audio content is: {response.audio_content}")
@@ -417,12 +420,11 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
                     response.audio_bytes,
                     speaker,
                 )
-
         if "text" in modalities:
             assert response.text_content is not None, "No text output is generated"
             print(f"text content is: {response.text_content}")
 
-        # Verify image description
+        # Verify keywords in output
         word_types = ["text", "image", "audio", "video"]
         keywords_dict = request_config.get("key_words", {})
         for word_type in word_types:
@@ -441,11 +443,25 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
                     )
 
         # Verify similarity (Whisper transcript vs streamed/detokenized text)
-        if "text" in modalities and "audio" in modalities:
-            assert response.similarity is not None and response.similarity > 0.9, (
-                "The audio content is not same as the text"
-            )
-            print(f"similarity is: {response.similarity}")
+        if "audio" in modalities:
+            audio_ref_text = request_config.get("audio_ref_text")
+            if "text" in modalities:
+                transcript = (response.audio_content or "").strip()
+                text_output = (response.text_content or "").strip()
+                similarity = cosine_similarity_text(
+                    transcript.lower(),
+                    text_output.lower(),
+                )
+                assert similarity > 0.9, "The audio content is not same as the text"
+                print(f"similarity is: {similarity}")
+            if audio_ref_text:
+                audio_similarity = cosine_similarity_text(
+                    response.audio_content.lower(),
+                    str(audio_ref_text).lower(),
+                )
+                assert audio_similarity > 0.9, (
+                    f"The audio content does not match reference text: similarity={audio_similarity:.3f}"
+                )
 
 
 def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str) -> None:
@@ -464,7 +480,7 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
     elif req_fmt == "wav" and response.audio_format:
         assert req_fmt in response.audio_format
 
-    if run_level == "advanced_model" and req_fmt != "pcm":
+    if run_level in {"advanced_model", "full_model"} and req_fmt != "pcm":
         expected_text = request_config.get("input")
         if expected_text:
             transcript = (response.audio_content or "").strip()

tests/helpers/fixtures/run_args.py

@@ -6,8 +6,8 @@ def pytest_addoption(parser):
         "--run-level",
         action="store",
         default="core_model",
-        choices=["core_model", "advanced_model"],
-        help="Test level to run: L2, L3",
+        choices=["core_model", "advanced_model", "full_model"],
+        help="Test level to run: L2, L3, L4",
     )
 
 

tests/helpers/fixtures/runtime.py

@@ -36,7 +36,7 @@ def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: st
         model = model_prefix + params.model
         port = params.port
         stage_config_path = params.stage_config_path
-        if run_level == "advanced_model" and stage_config_path is not None:
+        if run_level in {"advanced_model", "full_model"} and stage_config_path is not None:
             with open(stage_config_path, encoding="utf-8") as f:
                 cfg = yaml.safe_load(f) or {}
             # Strip ``load_format: dummy`` (CI overlay default) so advanced_model

tests/helpers/runtime.py

@@ -35,7 +35,6 @@ from tests.helpers.env import run_forced_gpu_cleanup_round
 from tests.helpers.media import (
     _merge_base64_audio_to_segment,
     convert_audio_bytes_to_text,
-    cosine_similarity_text,
     decode_b64_image,
 )
 from vllm_omni.config.stage_config import resolve_deploy_yaml
@@ -500,7 +499,6 @@ class OmniResponse:
     audio_content: str | None = None
     audio_format: str | None = None
     audio_bytes: bytes | None = None
-    similarity: float | None = None
     e2e_latency: float | None = None
     success: bool = False
     error_message: str | None = None
@@ -542,19 +540,15 @@ class OpenAIClientHandler:
                         text_content += content
             result.e2e_latency = time.perf_counter() - start_time
             audio_content = None
-            similarity = None
             if audio_data:
                 merged_seg = _merge_base64_audio_to_segment(audio_data)
                 wav_buf = BytesIO()
                 merged_seg.export(wav_buf, format="wav")
                 result.audio_bytes = wav_buf.getvalue()
                 audio_content = convert_audio_bytes_to_text(result.audio_bytes)
-            if audio_content and text_content:
-                similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
             result.text_content = text_content
             result.audio_data = audio_data
             result.audio_content = audio_content
-            result.similarity = similarity
             result.success = True
         except Exception as e:
             result.error_message = f"Stream processing error: {str(e)}"
@@ -578,15 +572,11 @@ class OpenAIClientHandler:
                 result.cached_tokens = details.cached_tokens
             result.e2e_latency = time.perf_counter() - start_time
             audio_content = None
-            similarity = None
             if audio_data:
                 result.audio_bytes = base64.b64decode(audio_data)
                 audio_content = convert_audio_bytes_to_text(result.audio_bytes)
-            if audio_content and text_content:
-                similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
             result.text_content = text_content
             result.audio_content = audio_content
-            result.similarity = similarity
             result.success = True
         except Exception as e:
             result.error_message = f"Non-stream processing error: {str(e)}"