From 26d481fc847a584c3f385a9ddcce002af1bbd319 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sun, 10 May 2026 07:00:29 +0800 Subject: [PATCH] [CI] Remove VLLM_TEST_CLEAN_GPU_MEMORY to avoid environment variable pollution that causes unnecessary GPU detection, thereby slowing down test case execution. (#3446) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] Co-authored-by: Hongsheng Liu --- .buildkite/test-amd-merge.yml | 4 ---- .buildkite/test-amd-ready.yaml | 4 ---- .buildkite/test-amd.yaml | 2 -- .buildkite/test-merge.yml | 21 ++-------------- .buildkite/test-nightly.yml | 1 - .buildkite/test-ready.yml | 24 ++----------------- docs/contributing/ci/tests_style.md | 1 - .../HunyuanImage-3.0-Instruct.md | 6 ++--- tests/dfx/perf/scripts/run_benchmark.py | 1 - .../perf/scripts/run_diffusion_benchmark.py | 1 - .../test_vae_decode_parallelism.py | 1 - .../test_bagel_mooncake_connector.py | 2 ++ .../test_bagel_shared_memory_connector.py | 20 ++-------------- .../accuracy/qwen3_omni/test_qwen3_omni.py | 1 - .../test_diffusers_backend_similarity.py | 8 +++---- .../accuracy/test_ltx2_3_video_similarity.py | 4 ++-- tests/e2e/accuracy/test_qwen_image.py | 4 ++-- tests/e2e/accuracy/test_qwen_image_edit.py | 4 ++-- tests/e2e/accuracy/test_qwen_image_layered.py | 4 ++-- .../test_dynin_omni_expansion.py | 1 - .../test_ming_flash_omni_expansion.py | 1 - .../test_omnivoice_expansion.py | 1 - .../e2e/offline_inference/test_qwen3_omni.py | 1 - ...st_qwen3_omni_autoround_w4a16_expansion.py | 1 - .../offline_inference/test_qwen3_tts_base.py | 1 - .../test_qwen3_tts_customvoice.py | 1 - tests/e2e/online_serving/test_bagel.py | 1 - .../test_bagel_multi_replicas.py | 1 - .../test_cosyvoice3_tts_expansion.py | 1 - .../test_dynin_omni_expansion.py | 1 - .../test_ming_flash_omni_expansion.py | 1 - .../test_moss_tts_nano_expansion.py | 1 - .../test_omnivoice_expansion.py | 1 - .../test_qwen2_5_omni_expansion.py | 1 - tests/e2e/online_serving/test_qwen3_omni.py | 1 - .../test_qwen3_omni_multi_replicas.py | 1 - .../e2e/online_serving/test_qwen3_tts_base.py | 1 - .../test_qwen3_tts_base_expansion.py | 1 - .../test_qwen3_tts_customvoice.py | 1 - .../test_qwen3_tts_customvoice_expansion.py | 1 - tests/e2e/online_serving/test_qwen_image.py | 1 - tests/e2e/online_serving/test_voxtral_tts.py | 1 - tests/e2e/online_serving/test_wan22_t2v.py | 1 - .../test_openai_audio_list_endpoints.py | 1 - .../test_qwen3_tts_speaker_embedding.py | 1 - .../openai_api/test_qwen3_tts_websocket.py | 1 - .../openai_api/test_serving_speech_batch.py | 1 - tests/entrypoints/test_omni_sleep_mode.py | 5 +++- tests/entrypoints/test_stage_utils.py | 3 --- .../offline_inference/test_internvla_a1.py | 5 ++-- .../offline_inference/test_text_to_image.py | 7 +++++- tests/helpers/env.py | 23 ++---------------- tests/helpers/fixtures/env.py | 8 ++++--- tests/helpers/runtime.py | 18 ++++++++------ 54 files changed, 54 insertions(+), 156 deletions(-) diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index 0ec578060..5c76ed742 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -123,7 +123,6 @@ steps: grade: Blocking commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model" - label: "Qwen3-TTS CustomVoice E2E Test" @@ -164,7 +163,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 @@ -177,7 +175,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 @@ -190,7 +187,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_IMAGE_FETCH_TIMEOUT=60 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/test-amd-ready.yaml index 954459691..c608b3837 100644 --- a/.buildkite/test-amd-ready.yaml +++ b/.buildkite/test-amd-ready.yaml @@ -114,7 +114,6 @@ steps: # commands: # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py # - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model" @@ -165,7 +164,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 @@ -178,7 +176,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - export VLLM_ROCM_USE_AITER_RMSNORM=0 @@ -191,7 +188,6 @@ steps: # grade: Blocking # commands: # - export GPU_ARCHS=gfx942 -# - export VLLM_TEST_CLEAN_GPU_MEMORY=1 # - export VLLM_IMAGE_FETCH_TIMEOUT=60 # - export VLLM_LOGGING_LEVEL=DEBUG # - export VLLM_WORKER_MULTIPROC_METHOD=spawn diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index a07aa0035..520c1bbae 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -100,7 +100,6 @@ steps: commands: - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_TEST_CLEAN_GPU_MEMORY="1" - export VLLM_ROCM_USE_AITER=0 - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py @@ -116,5 +115,4 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export VLLM_TEST_CLEAN_GPU_MEMORY="1" - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model" diff --git a/.buildkite/test-merge.yml b/.buildkite/test-merge.yml index 4f4f22338..c08d3f333 100644 --- a/.buildkite/test-merge.yml +++ b/.buildkite/test-merge.yml @@ -7,25 +7,9 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-merge-pipeline steps: - - label: "Simple · Diffusion Test" + - label: "Simple · Diffusion & Model Executor Test" commands: - - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - - label: "Simple · Model Executor Test" - commands: - - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" agents: queue: "gpu_1_queue" plugins: @@ -81,7 +65,6 @@ steps: - | timeout 55m bash -c " set -e - export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model' " diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 51d1bfa55..f46e17aec 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -534,7 +534,6 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" timeout_in_minutes: 60 commands: - - export VLLM_TEST_CLEAN_GPU_MEMORY="1" - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model" agents: queue: "mithril-h100-pool" diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 4da1ed988..f36b01e13 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -7,25 +7,9 @@ steps: - group: ":card_index_dividers: Simple Test" depends_on: upload-ready-pipeline steps: - - label: "Simple · Diffusion Test" + - label: "Simple · Diffusion & Model Executor Test" commands: - - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" - agents: - queue: "gpu_1_queue" - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - - label: "Simple · Model Executor Test" - commands: - - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" + - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" agents: queue: "gpu_1_queue" plugins: @@ -374,7 +358,6 @@ steps: commands: - | timeout 40m bash -c " - export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model' " @@ -416,7 +399,6 @@ steps: commands: - | timeout 40m bash -c " - export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' " @@ -459,7 +441,6 @@ steps: commands: - | timeout 40m bash -c " - export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' " @@ -501,7 +482,6 @@ steps: commands: - | timeout 40m bash -c " - export VLLM_TEST_CLEAN_GPU_MEMORY=1 export VLLM_IMAGE_FETCH_TIMEOUT=60 pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model' " diff --git a/docs/contributing/ci/tests_style.md b/docs/contributing/ci/tests_style.md index 9befda282..420a8821d 100644 --- a/docs/contributing/ci/tests_style.md +++ b/docs/contributing/ci/tests_style.md @@ -213,7 +213,6 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import threading from pathlib import Path diff --git a/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md b/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md index 2ef99317e..1694cc241 100644 --- a/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md +++ b/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md @@ -186,9 +186,9 @@ Validated score summary: The CI assertion threshold is `overall_mean >= 0.45`, so the smoke result is comfortably above the gate. The generate server and judge server run -sequentially through the `OmniServer` fixture, and -`VLLM_TEST_CLEAN_GPU_MEMORY=1` is used to wait for memory cleanup between -server lifetimes. +sequentially through the `OmniServer` fixture, with GPU memory cleanup +between server lifetimes (for example via the `clean_gpu_memory_between_tests` +pytest fixture in the smoke path). The lower-cost 2-GPU Instruct setup was tried for this smoke path but did not fit in memory. A previous 2-GPU experiment used the base HunyuanImage-3.0 diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 6dbce02b0..e129e9212 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -21,7 +21,6 @@ pytestmark = [pytest.mark.full_model] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" def _get_config_file_from_argv() -> str | None: diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index f33cacc07..639f03623 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -36,7 +36,6 @@ import pytest pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") # --------------------------------------------------------------------------- diff --git a/tests/diffusion/distributed/test_vae_decode_parallelism.py b/tests/diffusion/distributed/test_vae_decode_parallelism.py index 32902c318..2fd531e3e 100644 --- a/tests/diffusion/distributed/test_vae_decode_parallelism.py +++ b/tests/diffusion/distributed/test_vae_decode_parallelism.py @@ -21,7 +21,6 @@ import time from tests.helpers.runtime import OmniRunner from vllm_omni.platforms import current_omni_platform -# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" all_model_configs = [ diff --git a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py index 795a99e7b..a308fe5ea 100644 --- a/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_mooncake_connector.py @@ -26,6 +26,8 @@ from tests.helpers.runtime import OmniRunner from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config from vllm_omni.entrypoints.omni import Omni +pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")] + BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml") # Reference pixel data extracted from the known-good output image diff --git a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py index cc59a1bd0..57c6a79fb 100644 --- a/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py +++ b/tests/distributed/omni_connectors/test_bagel_shared_memory_connector.py @@ -26,6 +26,8 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf from vllm_omni.entrypoints.omni import Omni from vllm_omni.platforms import current_omni_platform +pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")] + BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml") # Reference pixel data extracted from the known-good output image @@ -234,24 +236,6 @@ def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT) return generated_image -def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT) -> Image.Image: - """Generate an image using Bagel text2img with configured parameters.""" - params_list = _configure_sampling_params(omni) - - omni_outputs = list( - omni.generate( - prompts=[{"prompt": prompt, "modalities": ["image"]}], - sampling_params_list=params_list, - ) - ) - - generated_image = _extract_generated_image(omni_outputs) - assert generated_image is not None, "No images generated" - assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}" - - return generated_image - - def _resolve_deploy_config(config_path: str, run_level: str) -> str: """Resolve deploy config based on run level. diff --git a/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py b/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py index 773f7c110..121551664 100644 --- a/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py +++ b/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py @@ -111,7 +111,6 @@ def _daily_omni_hub_inline_media(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped) monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") - monkeypatch.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) diff --git a/tests/e2e/accuracy/test_diffusers_backend_similarity.py b/tests/e2e/accuracy/test_diffusers_backend_similarity.py index 6392d9bcf..b06258a92 100644 --- a/tests/e2e/accuracy/test_diffusers_backend_similarity.py +++ b/tests/e2e/accuracy/test_diffusers_backend_similarity.py @@ -119,7 +119,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage] from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() apply_ftfy_mock() pipe: WanImageToVideoPipeline | None = None try: @@ -161,7 +161,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]: @@ -206,7 +206,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]: - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() pipe: DiffusionPipeline | None = None try: pipe = DiffusionPipeline.from_pretrained( @@ -241,7 +241,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() @pytest.mark.benchmark diff --git a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py b/tests/e2e/accuracy/test_ltx2_3_video_similarity.py index 752e78847..7fb2297a2 100644 --- a/tests/e2e/accuracy/test_ltx2_3_video_similarity.py +++ b/tests/e2e/accuracy/test_ltx2_3_video_similarity.py @@ -172,7 +172,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]: """Generate video using stock diffusers LTX2Pipeline.""" from diffusers import LTX2Pipeline - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() pipe = None try: pipe = LTX2Pipeline.from_pretrained( @@ -200,7 +200,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]: gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() # --------------------------------------------------------------------------- diff --git a/tests/e2e/accuracy/test_qwen_image.py b/tests/e2e/accuracy/test_qwen_image.py index 7ddf0fe6c..97f721139 100644 --- a/tests/e2e/accuracy/test_qwen_image.py +++ b/tests/e2e/accuracy/test_qwen_image.py @@ -70,7 +70,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image: def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image: - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() pipe: DiffusionPipeline | None = None try: pipe = DiffusionPipeline.from_pretrained( @@ -99,7 +99,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image: gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() @pytest.mark.benchmark diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index fd733b9de..a910ed5ba 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -77,7 +77,7 @@ def _run_diffusers_image_edit( input_images: list[Image.Image], output_path: Path, ) -> Image.Image: - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None device = torch.device("cuda:0") torch.cuda.set_device(device) @@ -110,7 +110,7 @@ def _run_diffusers_image_edit( gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() def _vllm_omni_output_single_image( diff --git a/tests/e2e/accuracy/test_qwen_image_layered.py b/tests/e2e/accuracy/test_qwen_image_layered.py index cd1c6fbc5..f2bc1b6aa 100644 --- a/tests/e2e/accuracy/test_qwen_image_layered.py +++ b/tests/e2e/accuracy/test_qwen_image_layered.py @@ -93,7 +93,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]: - run_pre_test_cleanup(enable_force=True) + run_pre_test_cleanup() pipe: DiffusionPipeline | None = None try: pipe = DiffusionPipeline.from_pretrained( @@ -126,7 +126,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o gc.collect() if torch.cuda.is_available(): torch.accelerator.empty_cache() - run_post_test_cleanup(enable_force=True) + run_post_test_cleanup() @pytest.mark.benchmark diff --git a/tests/e2e/offline_inference/test_dynin_omni_expansion.py b/tests/e2e/offline_inference/test_dynin_omni_expansion.py index 5d69bba93..7aae111f0 100644 --- a/tests/e2e/offline_inference/test_dynin_omni_expansion.py +++ b/tests/e2e/offline_inference/test_dynin_omni_expansion.py @@ -22,7 +22,6 @@ from tests.helpers.mark import hardware_test from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" _REPO_ROOT = Path(__file__).resolve().parents[3] _DEFAULT_DYNIN_CONFIG_PATH: Path | None = None diff --git a/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py b/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py index 6a10e1415..a81cfe6d4 100644 --- a/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py +++ b/tests/e2e/offline_inference/test_ming_flash_omni_expansion.py @@ -4,7 +4,6 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/offline_inference/test_omnivoice_expansion.py b/tests/e2e/offline_inference/test_omnivoice_expansion.py index 5319785e8..7ea646a85 100644 --- a/tests/e2e/offline_inference/test_omnivoice_expansion.py +++ b/tests/e2e/offline_inference/test_omnivoice_expansion.py @@ -9,7 +9,6 @@ Uses GPUGenerationWorker for both stages (iterative unmasking + DAC decoder). import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import numpy as np import pytest diff --git a/tests/e2e/offline_inference/test_qwen3_omni.py b/tests/e2e/offline_inference/test_qwen3_omni.py index a7c07235f..22225e8b2 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni.py +++ b/tests/e2e/offline_inference/test_qwen3_omni.py @@ -5,7 +5,6 @@ E2E offline tests for Omni model with video input and audio output. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py b/tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py index 046854e77..6cd630e77 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py +++ b/tests/e2e/offline_inference/test_qwen3_omni_autoround_w4a16_expansion.py @@ -50,7 +50,6 @@ def _qwen3_omni_env(): """ with pytest.MonkeyPatch.context() as mp: mp.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") - mp.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") yield diff --git a/tests/e2e/offline_inference/test_qwen3_tts_base.py b/tests/e2e/offline_inference/test_qwen3_tts_base.py index 5e053c046..ecd769166 100644 --- a/tests/e2e/offline_inference/test_qwen3_tts_base.py +++ b/tests/e2e/offline_inference/test_qwen3_tts_base.py @@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/offline_inference/test_qwen3_tts_customvoice.py b/tests/e2e/offline_inference/test_qwen3_tts_customvoice.py index 23b83b6ea..416a5009c 100644 --- a/tests/e2e/offline_inference/test_qwen3_tts_customvoice.py +++ b/tests/e2e/offline_inference/test_qwen3_tts_customvoice.py @@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_bagel.py b/tests/e2e/online_serving/test_bagel.py index a8ec65489..b22762acd 100644 --- a/tests/e2e/online_serving/test_bagel.py +++ b/tests/e2e/online_serving/test_bagel.py @@ -32,7 +32,6 @@ from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "ByteDance-Seed/BAGEL-7B-MoT" STAGE_CONFIGS_PATH = get_deploy_config_path("ci/bagel.yaml") diff --git a/tests/e2e/online_serving/test_bagel_multi_replicas.py b/tests/e2e/online_serving/test_bagel_multi_replicas.py index ba6ae6953..cb7aeeb94 100644 --- a/tests/e2e/online_serving/test_bagel_multi_replicas.py +++ b/tests/e2e/online_serving/test_bagel_multi_replicas.py @@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "ByteDance-Seed/BAGEL-7B-MoT" MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml") diff --git a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py index cd571f904..bf347db6f 100644 --- a/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py +++ b/tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py @@ -10,7 +10,6 @@ the CosyVoice3 model, which requires reference audio for voice cloning. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_dynin_omni_expansion.py b/tests/e2e/online_serving/test_dynin_omni_expansion.py index 19f0ef30b..702ff7f94 100644 --- a/tests/e2e/online_serving/test_dynin_omni_expansion.py +++ b/tests/e2e/online_serving/test_dynin_omni_expansion.py @@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.full_model, pytest.mark.omni] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "snu-aidas/Dynin-Omni" STAGE_CONFIG = get_deploy_config_path("dynin_omni_ci.yaml") diff --git a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py index 60f61233a..8ab327022 100644 --- a/tests/e2e/online_serving/test_ming_flash_omni_expansion.py +++ b/tests/e2e/online_serving/test_ming_flash_omni_expansion.py @@ -19,7 +19,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" pytestmark = [pytest.mark.omni, pytest.mark.full_model] diff --git a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py index e37c92ca3..d10dd6f57 100644 --- a/tests/e2e/online_serving/test_moss_tts_nano_expansion.py +++ b/tests/e2e/online_serving/test_moss_tts_nano_expansion.py @@ -17,7 +17,6 @@ import os import urllib.request os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_omnivoice_expansion.py b/tests/e2e/online_serving/test_omnivoice_expansion.py index cc048ac54..7ad3b9b25 100644 --- a/tests/e2e/online_serving/test_omnivoice_expansion.py +++ b/tests/e2e/online_serving/test_omnivoice_expansion.py @@ -10,7 +10,6 @@ accessed through the standard OpenAI-compatible speech API. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py index a325952c0..c49b42aa7 100644 --- a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py @@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" models = ["Qwen/Qwen2.5-Omni-7B"] diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index f1a3ce353..82c2f902a 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -13,7 +13,6 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf from vllm_omni.platforms import current_omni_platform os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] diff --git a/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py b/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py index 79a3e8893..f269836f3 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py +++ b/tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py @@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniResponse, OmniServerParams, dummy_messages from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct" MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe_multi_replicas_4gpu.yaml") diff --git a/tests/e2e/online_serving/test_qwen3_tts_base.py b/tests/e2e/online_serving/test_qwen3_tts_base.py index b577dae98..87c25f5b6 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base.py @@ -10,7 +10,6 @@ actual model inference, not mocks. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py index 2643f6d49..a4267c096 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_base_expansion.py @@ -19,7 +19,6 @@ from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.full_model, pytest.mark.tts] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py index c4273fd12..06bb7b44b 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice.py @@ -10,7 +10,6 @@ actual model inference, not mocks. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py index 7d374d770..9d9da2847 100644 --- a/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py @@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.full_model, pytest.mark.tts] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" diff --git a/tests/e2e/online_serving/test_qwen_image.py b/tests/e2e/online_serving/test_qwen_image.py index 23724157c..454a87aa5 100644 --- a/tests/e2e/online_serving/test_qwen_image.py +++ b/tests/e2e/online_serving/test_qwen_image.py @@ -27,7 +27,6 @@ from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Qwen/Qwen-Image" T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." diff --git a/tests/e2e/online_serving/test_voxtral_tts.py b/tests/e2e/online_serving/test_voxtral_tts.py index a33f881a0..dff53d539 100644 --- a/tests/e2e/online_serving/test_voxtral_tts.py +++ b/tests/e2e/online_serving/test_voxtral_tts.py @@ -10,7 +10,6 @@ actual model inference, not mocks. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import pytest diff --git a/tests/e2e/online_serving/test_wan22_t2v.py b/tests/e2e/online_serving/test_wan22_t2v.py index f220111f5..e6fe38086 100644 --- a/tests/e2e/online_serving/test_wan22_t2v.py +++ b/tests/e2e/online_serving/test_wan22_t2v.py @@ -21,7 +21,6 @@ from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage." diff --git a/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py b/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py index ca1954720..32a2dedbd 100644 --- a/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py +++ b/tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py @@ -9,7 +9,6 @@ Migrated from ``tests/e2e/online_serving/test_voxtral_tts.py`` to colocate with import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import httpx import pytest diff --git a/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py b/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py index 5b76dfabe..48e3670b4 100644 --- a/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py +++ b/tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py @@ -10,7 +10,6 @@ via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import struct diff --git a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py index e808dc678..296138564 100644 --- a/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py +++ b/tests/entrypoints/openai_api/test_qwen3_tts_websocket.py @@ -16,7 +16,6 @@ from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" diff --git a/tests/entrypoints/openai_api/test_serving_speech_batch.py b/tests/entrypoints/openai_api/test_serving_speech_batch.py index 6ce971672..054100ce8 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_batch.py +++ b/tests/entrypoints/openai_api/test_serving_speech_batch.py @@ -9,7 +9,6 @@ Validates bulk synthesis via the batch API with actual model inference. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import base64 import struct diff --git a/tests/entrypoints/test_omni_sleep_mode.py b/tests/entrypoints/test_omni_sleep_mode.py index 1fc0794b2..1365b320d 100644 --- a/tests/entrypoints/test_omni_sleep_mode.py +++ b/tests/entrypoints/test_omni_sleep_mode.py @@ -18,7 +18,10 @@ from vllm_omni.platforms import current_omni_platform logging.basicConfig(level=logging.INFO) logger = logging.getLogger("OmniTest") -pytestmark = [pytest.mark.advanced_model] +pytestmark = [ + pytest.mark.advanced_model, + pytest.mark.usefixtures("clean_gpu_memory_between_tests"), +] def clean_gpu_envs(): diff --git a/tests/entrypoints/test_stage_utils.py b/tests/entrypoints/test_stage_utils.py index 15ee9c32a..4bc26bff9 100644 --- a/tests/entrypoints/test_stage_utils.py +++ b/tests/entrypoints/test_stage_utils.py @@ -55,7 +55,6 @@ def _make_mock_platform(mocker, device_type: str = "cuda", env_var: str = "CUDA_ @pytest.mark.core_model @pytest.mark.cpu -@pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): # Preserve an existing logical mapping and ensure devices "0,1" map through it. monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7") @@ -77,7 +76,6 @@ def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypat @pytest.mark.core_model @pytest.mark.cpu -@pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): # Preserve an existing logical mapping and ensure devices "0,1" map through it. monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7") @@ -98,7 +96,6 @@ def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, mon assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7" -@pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): """Test that set_stage_devices works correctly for NPU platform.""" monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "4,5") diff --git a/tests/examples/offline_inference/test_internvla_a1.py b/tests/examples/offline_inference/test_internvla_a1.py index 38104dc66..791d380f0 100644 --- a/tests/examples/offline_inference/test_internvla_a1.py +++ b/tests/examples/offline_inference/test_internvla_a1.py @@ -16,9 +16,9 @@ import pytest from tests.helpers.mark import hardware_test -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" - REPO_ROOT = Path(__file__).resolve().parents[3] + + EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py" @@ -33,6 +33,7 @@ def _required_env(name: str) -> str: @pytest.mark.advanced_model @pytest.mark.diffusion @hardware_test(res={"cuda": "H100", "rocm": "MI325"}) +@pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_internvla_a1_offline_open_loop(run_level: str) -> None: if run_level != "advanced_model": pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.") diff --git a/tests/examples/offline_inference/test_text_to_image.py b/tests/examples/offline_inference/test_text_to_image.py index a63af7cfb..701f1a184 100644 --- a/tests/examples/offline_inference/test_text_to_image.py +++ b/tests/examples/offline_inference/test_text_to_image.py @@ -11,7 +11,12 @@ from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet from tests.helpers.assertions import assert_image_valid from tests.helpers.mark import hardware_marks -pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})] +pytestmark = [ + pytest.mark.usefixtures("clean_gpu_memory_between_tests"), + pytest.mark.full_model, + pytest.mark.example, + *hardware_marks(res={"cuda": "H100"}), +] T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py" README_PATH = T2I_SCRIPT.with_name("README.md") diff --git a/tests/helpers/env.py b/tests/helpers/env.py index d6f682659..7e7e9cc93 100644 --- a/tests/helpers/env.py +++ b/tests/helpers/env.py @@ -16,11 +16,6 @@ from contextlib import contextmanager import torch -def run_forced_gpu_cleanup_round() -> None: - run_pre_test_cleanup(enable_force=True) - run_post_test_cleanup(enable_force=True) - - def get_physical_device_indices(devices): visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") if visible_devices is None: @@ -232,16 +227,7 @@ def _print_gpu_processes() -> None: print("=" * 80) -_SKIPPED_GPU_CLEANUP_MSG = ( - "\nSkipping GPU memory cleanup check (typically: instance already up; no check needed between tests)\n" -) - - -def run_pre_test_cleanup(enable_force: bool = False) -> None: - if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force: - print(_SKIPPED_GPU_CLEANUP_MSG) - return - +def run_pre_test_cleanup() -> None: print("Pre-test GPU status:") num_gpus = torch.accelerator.device_count() @@ -255,11 +241,7 @@ def run_pre_test_cleanup(enable_force: bool = False) -> None: print(f"Pre-test cleanup note: {e}") -def run_post_test_cleanup(enable_force: bool = False) -> None: - if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force: - print(_SKIPPED_GPU_CLEANUP_MSG) - return - +def run_post_test_cleanup() -> None: if torch.cuda.is_available(): gc.collect() torch.accelerator.empty_cache() @@ -318,6 +300,5 @@ __all__ = [ "get_physical_device_indices", "run_post_test_cleanup", "run_pre_test_cleanup", - "run_forced_gpu_cleanup_round", "wait_for_gpu_memory_to_clear", ] diff --git a/tests/helpers/fixtures/env.py b/tests/helpers/fixtures/env.py index aa7a2fd04..452ae2d2d 100644 --- a/tests/helpers/fixtures/env.py +++ b/tests/helpers/fixtures/env.py @@ -27,10 +27,12 @@ def model_prefix() -> str: return f"{prefix.rstrip('/')}/" if prefix else "" -@pytest.fixture(autouse=True) +@pytest.fixture def clean_gpu_memory_between_tests(): - # Import here so ``tests.helpers.env`` (and vLLM platform modules) load only - # after session autouse fixtures like ``default_env`` have run (RFC #2299). + """Opt-in GPU pre/post hooks for a test (no environment-variable gate). + + Use as a test parameter or ``@pytest.mark.usefixtures("clean_gpu_memory_between_tests")``. + """ from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup print("\n=== PRE-TEST GPU CLEANUP ===") diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index eb1158d6e..a50390d6c 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -35,7 +35,7 @@ from tests.helpers.assertions import ( assert_diffusion_response, assert_omni_response, ) -from tests.helpers.env import run_forced_gpu_cleanup_round +from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.media import ( _merge_base64_audio_to_segment, convert_audio_bytes_to_text, @@ -201,7 +201,8 @@ class OmniServer: env_dict: dict[str, str] | None = None, use_omni: bool = True, ) -> None: - run_forced_gpu_cleanup_round() + run_pre_test_cleanup() + run_post_test_cleanup() cleanup_dist_env_and_memory() self.model = model self.serve_args = serve_args @@ -306,7 +307,8 @@ class OmniServer: def __exit__(self, exc_type, exc_val, exc_tb): if self.proc: self._kill_process_tree(self.proc.pid) - run_forced_gpu_cleanup_round() + run_pre_test_cleanup() + run_post_test_cleanup() cleanup_dist_env_and_memory() @@ -593,7 +595,8 @@ class OmniServerStageCli(OmniServer): proc = self.stage_procs[stage_key] if proc.poll() is None: self._kill_process_tree(proc.pid) - run_forced_gpu_cleanup_round() + run_pre_test_cleanup() + run_post_test_cleanup() cleanup_dist_env_and_memory() @@ -1272,7 +1275,8 @@ class OmniRunner: **kwargs, ) -> None: cleanup_dist_env_and_memory() - run_forced_gpu_cleanup_round() + run_pre_test_cleanup() + run_post_test_cleanup() self.model_name = model_name self.seed = seed self._prompt_len_estimate_cache: dict[str, Any] = {} @@ -1538,7 +1542,8 @@ class OmniRunner: if hasattr(self.omni, "close"): self.omni.close() self._cleanup_process() - run_forced_gpu_cleanup_round() + run_pre_test_cleanup() + run_post_test_cleanup() cleanup_dist_env_and_memory() @@ -1876,6 +1881,5 @@ __all__ = [ "OmniServerStageCli", "OpenAIClientHandler", "get_open_port", - "run_forced_gpu_cleanup_round", "dummy_messages_from_mix_data", ]