[CI] Remove VLLM_TEST_CLEAN_GPU_MEMORY to avoid environment variable pollution that causes unnecessary GPU detection, thereby slowing down test case execution. (#3446)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
wangyuandHongsheng Liu authored and GitHub committed 2026-05-10 07:00:29 +08:00
1 parent 77480215f5
commit 26d481fc84
54 files changed
+54 -156

No files matched your search

-4
View File
@@ -123,7 +123,6 @@ steps:
grade: Blocking
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
- label: "Qwen3-TTS CustomVoice E2E Test"
@@ -164,7 +163,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
@@ -177,7 +175,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
@@ -190,7 +187,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-4
View File
@@ -114,7 +114,6 @@ steps:
# commands:
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
# - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
@@ -165,7 +164,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
@@ -178,7 +176,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
@@ -191,7 +188,6 @@ steps:
# grade: Blocking
# commands:
# - export GPU_ARCHS=gfx942
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
# - export VLLM_LOGGING_LEVEL=DEBUG
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-2
View File
@@ -100,7 +100,6 @@ steps:
commands:
- export VLLM_LOGGING_LEVEL=DEBUG
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
- export VLLM_ROCM_USE_AITER=0
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
- pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
@@ -116,5 +115,4 @@ steps:
- export GPU_ARCHS=gfx942
- export VLLM_LOGGING_LEVEL=DEBUG
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
- pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
+2 -19
View File
@@ -7,25 +7,9 @@ steps:
- group: ":card_index_dividers: Simple Test"
depends_on: upload-merge-pipeline
steps:
- label: "Simple · Diffusion Test"
- label: "Simple · Diffusion & Model Executor Test"
commands:
- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
agents:
queue: "gpu_1_queue"
plugins:
- docker#v5.2.0:
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
always-pull: true
propagate-environment: true
environment:
- "HF_HOME=/fsx/hf_cache"
- "HF_TOKEN"
volumes:
- "/fsx/hf_cache:/fsx/hf_cache"
- label: "Simple · Model Executor Test"
commands:
- "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
agents:
queue: "gpu_1_queue"
plugins:
@@ -81,7 +65,6 @@ steps:
- |
timeout 55m bash -c "
set -e
export VLLM_TEST_CLEAN_GPU_MEMORY=1
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
"
-1
View File
@@ -534,7 +534,6 @@ steps:
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
timeout_in_minutes: 60
commands:
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
- pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
agents:
queue: "mithril-h100-pool"
+2 -22
View File
@@ -7,25 +7,9 @@ steps:
- group: ":card_index_dividers: Simple Test"
depends_on: upload-ready-pipeline
steps:
- label: "Simple · Diffusion Test"
- label: "Simple · Diffusion & Model Executor Test"
commands:
- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
agents:
queue: "gpu_1_queue"
plugins:
- docker#v5.2.0:
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
always-pull: true
propagate-environment: true
environment:
- "HF_HOME=/fsx/hf_cache"
- "HF_TOKEN"
volumes:
- "/fsx/hf_cache:/fsx/hf_cache"
- label: "Simple · Model Executor Test"
commands:
- "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
agents:
queue: "gpu_1_queue"
plugins:
@@ -374,7 +358,6 @@ steps:
commands:
- |
timeout 40m bash -c "
export VLLM_TEST_CLEAN_GPU_MEMORY=1
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
"
@@ -416,7 +399,6 @@ steps:
commands:
- |
timeout 40m bash -c "
export VLLM_TEST_CLEAN_GPU_MEMORY=1
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
"
@@ -459,7 +441,6 @@ steps:
commands:
- |
timeout 40m bash -c "
export VLLM_TEST_CLEAN_GPU_MEMORY=1
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
"
@@ -501,7 +482,6 @@ steps:
commands:
- |
timeout 40m bash -c "
export VLLM_TEST_CLEAN_GPU_MEMORY=1
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
"
-1
View File
@@ -213,7 +213,6 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import threading
from pathlib import Path
@@ -186,9 +186,9 @@ Validated score summary:
The CI assertion threshold is `overall_mean >= 0.45`, so the smoke result is
comfortably above the gate. The generate server and judge server run
sequentially through the `OmniServer` fixture, and
`VLLM_TEST_CLEAN_GPU_MEMORY=1` is used to wait for memory cleanup between
server lifetimes.
sequentially through the `OmniServer` fixture, with GPU memory cleanup
between server lifetimes (for example via the `clean_gpu_memory_between_tests`
pytest fixture in the smoke path).
The lower-cost 2-GPU Instruct setup was tried for this smoke path but did not
fit in memory. A previous 2-GPU experiment used the base HunyuanImage-3.0
-1
View File
@@ -21,7 +21,6 @@ pytestmark = [pytest.mark.full_model]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
def _get_config_file_from_argv() -> str | None:
@@ -36,7 +36,6 @@ import pytest
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
# ---------------------------------------------------------------------------
@@ -21,7 +21,6 @@ import time
from tests.helpers.runtime import OmniRunner
from vllm_omni.platforms import current_omni_platform
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
all_model_configs = [
@@ -26,6 +26,8 @@ from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.entrypoints.omni import Omni
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
# Reference pixel data extracted from the known-good output image
@@ -26,6 +26,8 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
# Reference pixel data extracted from the known-good output image
@@ -234,24 +236,6 @@ def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT)
return generated_image
def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT) -> Image.Image:
"""Generate an image using Bagel text2img with configured parameters."""
params_list = _configure_sampling_params(omni)
omni_outputs = list(
omni.generate(
prompts=[{"prompt": prompt, "modalities": ["image"]}],
sampling_params_list=params_list,
)
)
generated_image = _extract_generated_image(omni_outputs)
assert generated_image is not None, "No images generated"
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
return generated_image
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
"""Resolve deploy config based on run level.
@@ -111,7 +111,6 @@ def _daily_omni_hub_inline_media(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
monkeypatch.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@@ -119,7 +119,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage]
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
apply_ftfy_mock()
pipe: WanImageToVideoPipeline | None = None
try:
@@ -161,7 +161,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
@@ -206,7 +206,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -241,7 +241,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
@pytest.mark.benchmark
@@ -172,7 +172,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]:
"""Generate video using stock diffusers LTX2Pipeline."""
from diffusers import LTX2Pipeline
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
pipe = None
try:
pipe = LTX2Pipeline.from_pretrained(
@@ -200,7 +200,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]:
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
# ---------------------------------------------------------------------------
+2 -2
View File
@@ -70,7 +70,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image:
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -99,7 +99,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
@pytest.mark.benchmark
+2 -2
View File
@@ -77,7 +77,7 @@ def _run_diffusers_image_edit(
input_images: list[Image.Image],
output_path: Path,
) -> Image.Image:
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None
device = torch.device("cuda:0")
torch.cuda.set_device(device)
@@ -110,7 +110,7 @@ def _run_diffusers_image_edit(
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
def _vllm_omni_output_single_image(
@@ -93,7 +93,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o
def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]:
run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -126,7 +126,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup(enable_force=True)
run_post_test_cleanup()
@pytest.mark.benchmark
@@ -22,7 +22,6 @@ from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_DEFAULT_DYNIN_CONFIG_PATH: Path | None = None
@@ -4,7 +4,6 @@
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -9,7 +9,6 @@ Uses GPUGenerationWorker for both stages (iterative unmasking + DAC decoder).
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import numpy as np
import pytest
@@ -5,7 +5,6 @@ E2E offline tests for Omni model with video input and audio output.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -50,7 +50,6 @@ def _qwen3_omni_env():
"""
with pytest.MonkeyPatch.context() as mp:
mp.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
mp.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
yield
@@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
-1
View File
@@ -32,7 +32,6 @@ from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
STAGE_CONFIGS_PATH = get_deploy_config_path("ci/bagel.yaml")
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
@@ -10,7 +10,6 @@ the CosyVoice3 model, which requires reference audio for voice cloning.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "snu-aidas/Dynin-Omni"
STAGE_CONFIG = get_deploy_config_path("dynin_omni_ci.yaml")
@@ -19,7 +19,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
@@ -17,7 +17,6 @@ import os
import urllib.request
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -10,7 +10,6 @@ accessed through the standard OpenAI-compatible speech API.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen2.5-Omni-7B"]
@@ -13,7 +13,6 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniResponse, OmniServerParams, dummy_messages
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe_multi_replicas_4gpu.yaml")
@@ -10,7 +10,6 @@ actual model inference, not mocks.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -19,7 +19,6 @@ from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -10,7 +10,6 @@ actual model inference, not mocks.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
@@ -27,7 +27,6 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen-Image"
T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
@@ -10,7 +10,6 @@ actual model inference, not mocks.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
@@ -21,7 +21,6 @@ from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
@@ -9,7 +9,6 @@ Migrated from ``tests/e2e/online_serving/test_voxtral_tts.py`` to colocate with
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import httpx
import pytest
@@ -10,7 +10,6 @@ via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import struct
@@ -16,7 +16,6 @@ from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
@@ -9,7 +9,6 @@ Validates bulk synthesis via the batch API with actual model inference.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import base64
import struct
+4 -1
View File
@@ -18,7 +18,10 @@ from vllm_omni.platforms import current_omni_platform
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("OmniTest")
pytestmark = [pytest.mark.advanced_model]
pytestmark = [
pytest.mark.advanced_model,
pytest.mark.usefixtures("clean_gpu_memory_between_tests"),
]
def clean_gpu_envs():
-3
View File
@@ -55,7 +55,6 @@ def _make_mock_platform(mocker, device_type: str = "cuda", env_var: str = "CUDA_
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
@@ -77,7 +76,6 @@ def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypat
@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
@@ -98,7 +96,6 @@ def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, mon
assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7"
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
"""Test that set_stage_devices works correctly for NPU platform."""
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "4,5")
@@ -16,9 +16,9 @@ import pytest
from tests.helpers.mark import hardware_test
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
REPO_ROOT = Path(__file__).resolve().parents[3]
EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py"
@@ -33,6 +33,7 @@ def _required_env(name: str) -> str:
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_internvla_a1_offline_open_loop(run_level: str) -> None:
if run_level != "advanced_model":
pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.")
@@ -11,7 +11,12 @@ from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet
from tests.helpers.assertions import assert_image_valid
from tests.helpers.mark import hardware_marks
pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
pytestmark = [
pytest.mark.usefixtures("clean_gpu_memory_between_tests"),
pytest.mark.full_model,
pytest.mark.example,
*hardware_marks(res={"cuda": "H100"}),
]
T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py"
README_PATH = T2I_SCRIPT.with_name("README.md")
+2 -21
View File
@@ -16,11 +16,6 @@ from contextlib import contextmanager
import torch
def run_forced_gpu_cleanup_round() -> None:
run_pre_test_cleanup(enable_force=True)
run_post_test_cleanup(enable_force=True)
def get_physical_device_indices(devices):
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
if visible_devices is None:
@@ -232,16 +227,7 @@ def _print_gpu_processes() -> None:
print("=" * 80)
_SKIPPED_GPU_CLEANUP_MSG = (
"\nSkipping GPU memory cleanup check (typically: instance already up; no check needed between tests)\n"
)
def run_pre_test_cleanup(enable_force: bool = False) -> None:
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force:
print(_SKIPPED_GPU_CLEANUP_MSG)
return
def run_pre_test_cleanup() -> None:
print("Pre-test GPU status:")
num_gpus = torch.accelerator.device_count()
@@ -255,11 +241,7 @@ def run_pre_test_cleanup(enable_force: bool = False) -> None:
print(f"Pre-test cleanup note: {e}")
def run_post_test_cleanup(enable_force: bool = False) -> None:
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force:
print(_SKIPPED_GPU_CLEANUP_MSG)
return
def run_post_test_cleanup() -> None:
if torch.cuda.is_available():
gc.collect()
torch.accelerator.empty_cache()
@@ -318,6 +300,5 @@ __all__ = [
"get_physical_device_indices",
"run_post_test_cleanup",
"run_pre_test_cleanup",
"run_forced_gpu_cleanup_round",
"wait_for_gpu_memory_to_clear",
]
+5 -3
View File
@@ -27,10 +27,12 @@ def model_prefix() -> str:
return f"{prefix.rstrip('/')}/" if prefix else ""
@pytest.fixture(autouse=True)
@pytest.fixture
def clean_gpu_memory_between_tests():
# Import here so ``tests.helpers.env`` (and vLLM platform modules) load only
# after session autouse fixtures like ``default_env`` have run (RFC #2299).
"""Opt-in GPU pre/post hooks for a test (no environment-variable gate).
Use as a test parameter or ``@pytest.mark.usefixtures("clean_gpu_memory_between_tests")``.
"""
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
print("\n=== PRE-TEST GPU CLEANUP ===")
+11 -7
View File
@@ -35,7 +35,7 @@ from tests.helpers.assertions import (
assert_diffusion_response,
assert_omni_response,
)
from tests.helpers.env import run_forced_gpu_cleanup_round
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.media import (
_merge_base64_audio_to_segment,
convert_audio_bytes_to_text,
@@ -201,7 +201,8 @@ class OmniServer:
env_dict: dict[str, str] | None = None,
use_omni: bool = True,
) -> None:
run_forced_gpu_cleanup_round()
run_pre_test_cleanup()
run_post_test_cleanup()
cleanup_dist_env_and_memory()
self.model = model
self.serve_args = serve_args
@@ -306,7 +307,8 @@ class OmniServer:
def __exit__(self, exc_type, exc_val, exc_tb):
if self.proc:
self._kill_process_tree(self.proc.pid)
run_forced_gpu_cleanup_round()
run_pre_test_cleanup()
run_post_test_cleanup()
cleanup_dist_env_and_memory()
@@ -593,7 +595,8 @@ class OmniServerStageCli(OmniServer):
proc = self.stage_procs[stage_key]
if proc.poll() is None:
self._kill_process_tree(proc.pid)
run_forced_gpu_cleanup_round()
run_pre_test_cleanup()
run_post_test_cleanup()
cleanup_dist_env_and_memory()
@@ -1272,7 +1275,8 @@ class OmniRunner:
**kwargs,
) -> None:
cleanup_dist_env_and_memory()
run_forced_gpu_cleanup_round()
run_pre_test_cleanup()
run_post_test_cleanup()
self.model_name = model_name
self.seed = seed
self._prompt_len_estimate_cache: dict[str, Any] = {}
@@ -1538,7 +1542,8 @@ class OmniRunner:
if hasattr(self.omni, "close"):
self.omni.close()
self._cleanup_process()
run_forced_gpu_cleanup_round()
run_pre_test_cleanup()
run_post_test_cleanup()
cleanup_dist_env_and_memory()
@@ -1876,6 +1881,5 @@ __all__ = [
"OmniServerStageCli",
"OpenAIClientHandler",
"get_open_port",
"run_forced_gpu_cleanup_round",
"dummy_messages_from_mix_data",
]