[CI] Remove VLLM_TEST_CLEAN_GPU_MEMORY to avoid environment variable pollution that causes unnecessary GPU detection, thereby slowing down test case execution. (#3446)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] <your.email@example.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
77480215f5
commit
26d481fc84
54 files changed
+54
-156
No files matched your search
@@ -123,7 +123,6 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
|
||||
|
||||
- label: "Qwen3-TTS CustomVoice E2E Test"
|
||||
@@ -164,7 +163,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
@@ -177,7 +175,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
@@ -190,7 +187,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
@@ -114,7 +114,6 @@ steps:
|
||||
# commands:
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
# - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
|
||||
|
||||
@@ -165,7 +164,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
@@ -178,7 +176,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
@@ -191,7 +188,6 @@ steps:
|
||||
# grade: Blocking
|
||||
# commands:
|
||||
# - export GPU_ARCHS=gfx942
|
||||
# - export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
# - export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# - export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
@@ -100,7 +100,6 @@ steps:
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
|
||||
@@ -116,5 +115,4 @@ steps:
|
||||
- export GPU_ARCHS=gfx942
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
|
||||
- pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
|
||||
@@ -7,25 +7,9 @@ steps:
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Simple · Diffusion Test"
|
||||
- label: "Simple · Diffusion & Model Executor Test"
|
||||
commands:
|
||||
- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Simple · Model Executor Test"
|
||||
commands:
|
||||
- "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
@@ -81,7 +65,6 @@ steps:
|
||||
- |
|
||||
timeout 55m bash -c "
|
||||
set -e
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
|
||||
@@ -534,7 +534,6 @@ steps:
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY="1"
|
||||
- pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
|
||||
@@ -7,25 +7,9 @@ steps:
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Simple · Diffusion Test"
|
||||
- label: "Simple · Diffusion & Model Executor Test"
|
||||
commands:
|
||||
- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Simple · Model Executor Test"
|
||||
commands:
|
||||
- "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
@@ -374,7 +358,6 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
@@ -416,7 +399,6 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
@@ -459,7 +441,6 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
@@ -501,7 +482,6 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
|
||||
@@ -213,7 +213,6 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
@@ -186,9 +186,9 @@ Validated score summary:
|
||||
|
||||
The CI assertion threshold is `overall_mean >= 0.45`, so the smoke result is
|
||||
comfortably above the gate. The generate server and judge server run
|
||||
sequentially through the `OmniServer` fixture, and
|
||||
`VLLM_TEST_CLEAN_GPU_MEMORY=1` is used to wait for memory cleanup between
|
||||
server lifetimes.
|
||||
sequentially through the `OmniServer` fixture, with GPU memory cleanup
|
||||
between server lifetimes (for example via the `clean_gpu_memory_between_tests`
|
||||
pytest fixture in the smoke path).
|
||||
|
||||
The lower-cost 2-GPU Instruct setup was tried for this smoke path but did not
|
||||
fit in memory. A previous 2-GPU experiment used the base HunyuanImage-3.0
|
||||
|
||||
@@ -21,7 +21,6 @@ pytestmark = [pytest.mark.full_model]
|
||||
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
|
||||
def _get_config_file_from_argv() -> str | None:
|
||||
|
||||
@@ -36,7 +36,6 @@ import pytest
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -21,7 +21,6 @@ import time
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
all_model_configs = [
|
||||
|
||||
@@ -26,6 +26,8 @@ from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||
|
||||
BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
|
||||
@@ -26,6 +26,8 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.usefixtures("clean_gpu_memory_between_tests")]
|
||||
|
||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
@@ -234,24 +236,6 @@ def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT)
|
||||
return generated_image
|
||||
|
||||
|
||||
def _generate_bagel_text2img(omni: Omni, prompt: str = TEXT2IMG_DEFAULT_PROMPT) -> Image.Image:
|
||||
"""Generate an image using Bagel text2img with configured parameters."""
|
||||
params_list = _configure_sampling_params(omni)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[{"prompt": prompt, "modalities": ["image"]}],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
generated_image = _extract_generated_image(omni_outputs)
|
||||
assert generated_image is not None, "No images generated"
|
||||
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
|
||||
|
||||
return generated_image
|
||||
|
||||
|
||||
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
"""Resolve deploy config based on run level.
|
||||
|
||||
|
||||
@@ -111,7 +111,6 @@ def _daily_omni_hub_inline_media(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
monkeypatch.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
|
||||
@@ -119,7 +119,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
|
||||
from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage]
|
||||
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
|
||||
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
apply_ftfy_mock()
|
||||
pipe: WanImageToVideoPipeline | None = None
|
||||
try:
|
||||
@@ -161,7 +161,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
|
||||
@@ -206,7 +206,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -241,7 +241,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -172,7 +172,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]:
|
||||
"""Generate video using stock diffusers LTX2Pipeline."""
|
||||
from diffusers import LTX2Pipeline
|
||||
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
pipe = None
|
||||
try:
|
||||
pipe = LTX2Pipeline.from_pretrained(
|
||||
@@ -200,7 +200,7 @@ def _run_diffusers_baseline(model: str, output_dir: Path) -> list[Image.Image]:
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -70,7 +70,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -99,7 +99,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -77,7 +77,7 @@ def _run_diffusers_image_edit(
|
||||
input_images: list[Image.Image],
|
||||
output_path: Path,
|
||||
) -> Image.Image:
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None
|
||||
device = torch.device("cuda:0")
|
||||
torch.cuda.set_device(device)
|
||||
@@ -110,7 +110,7 @@ def _run_diffusers_image_edit(
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
def _vllm_omni_output_single_image(
|
||||
|
||||
@@ -93,7 +93,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]:
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_pre_test_cleanup()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -126,7 +126,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -22,7 +22,6 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_DEFAULT_DYNIN_CONFIG_PATH: Path | None = None
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ Uses GPUGenerationWorker for both stages (iterative unmasking + DAC decoder).
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
@@ -5,7 +5,6 @@ E2E offline tests for Omni model with video input and audio output.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -50,7 +50,6 @@ def _qwen3_omni_env():
|
||||
"""
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
mp.setenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
|
||||
yield
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ Same structure as test_qwen3_omni (models, stage_configs, test_params, parametri
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -32,7 +32,6 @@ from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
STAGE_CONFIGS_PATH = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
|
||||
|
||||
@@ -10,7 +10,6 @@ the CosyVoice3 model, which requires reference audio for voice cloning.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "snu-aidas/Dynin-Omni"
|
||||
STAGE_CONFIG = get_deploy_config_path("dynin_omni_ci.yaml")
|
||||
|
||||
@@ -19,7 +19,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ import os
|
||||
import urllib.request
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ accessed through the standard OpenAI-compatible speech API.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
|
||||
@@ -13,7 +13,6 @@ from tests.helpers.stage_config import get_deploy_config_path, modify_stage_conf
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
@@ -12,7 +12,6 @@ from tests.helpers.runtime import OmniResponse, OmniServerParams, dummy_messages
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe_multi_replicas_4gpu.yaml")
|
||||
|
||||
@@ -10,7 +10,6 @@ actual model inference, not mocks.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ actual model inference, not mocks.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen-Image"
|
||||
T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
|
||||
|
||||
@@ -10,7 +10,6 @@ actual model inference, not mocks.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -21,7 +21,6 @@ from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
|
||||
|
||||
@@ -9,7 +9,6 @@ Migrated from ``tests/e2e/online_serving/test_voxtral_tts.py`` to colocate with
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
@@ -10,7 +10,6 @@ via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import struct
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ Validates bulk synthesis via the batch API with actual model inference.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import base64
|
||||
import struct
|
||||
|
||||
@@ -18,7 +18,10 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger("OmniTest")
|
||||
pytestmark = [pytest.mark.advanced_model]
|
||||
pytestmark = [
|
||||
pytest.mark.advanced_model,
|
||||
pytest.mark.usefixtures("clean_gpu_memory_between_tests"),
|
||||
]
|
||||
|
||||
|
||||
def clean_gpu_envs():
|
||||
|
||||
@@ -55,7 +55,6 @@ def _make_mock_platform(mocker, device_type: str = "cuda", env_var: str = "CUDA_
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
|
||||
def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
|
||||
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
|
||||
@@ -77,7 +76,6 @@ def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypat
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
|
||||
def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
|
||||
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
|
||||
@@ -98,7 +96,6 @@ def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, mon
|
||||
assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
|
||||
def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that set_stage_devices works correctly for NPU platform."""
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "4,5")
|
||||
|
||||
@@ -16,9 +16,9 @@ import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py"
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ def _required_env(name: str) -> str:
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
|
||||
def test_internvla_a1_offline_open_loop(run_level: str) -> None:
|
||||
if run_level != "advanced_model":
|
||||
pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.")
|
||||
|
||||
@@ -11,7 +11,12 @@ from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet
|
||||
from tests.helpers.assertions import assert_image_valid
|
||||
from tests.helpers.mark import hardware_marks
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]
|
||||
pytestmark = [
|
||||
pytest.mark.usefixtures("clean_gpu_memory_between_tests"),
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.example,
|
||||
*hardware_marks(res={"cuda": "H100"}),
|
||||
]
|
||||
|
||||
T2I_SCRIPT = EXAMPLES / "offline_inference" / "text_to_image" / "text_to_image.py"
|
||||
README_PATH = T2I_SCRIPT.with_name("README.md")
|
||||
|
||||
+2
-21
@@ -16,11 +16,6 @@ from contextlib import contextmanager
|
||||
import torch
|
||||
|
||||
|
||||
def run_forced_gpu_cleanup_round() -> None:
|
||||
run_pre_test_cleanup(enable_force=True)
|
||||
run_post_test_cleanup(enable_force=True)
|
||||
|
||||
|
||||
def get_physical_device_indices(devices):
|
||||
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if visible_devices is None:
|
||||
@@ -232,16 +227,7 @@ def _print_gpu_processes() -> None:
|
||||
print("=" * 80)
|
||||
|
||||
|
||||
_SKIPPED_GPU_CLEANUP_MSG = (
|
||||
"\nSkipping GPU memory cleanup check (typically: instance already up; no check needed between tests)\n"
|
||||
)
|
||||
|
||||
|
||||
def run_pre_test_cleanup(enable_force: bool = False) -> None:
|
||||
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force:
|
||||
print(_SKIPPED_GPU_CLEANUP_MSG)
|
||||
return
|
||||
|
||||
def run_pre_test_cleanup() -> None:
|
||||
print("Pre-test GPU status:")
|
||||
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
@@ -255,11 +241,7 @@ def run_pre_test_cleanup(enable_force: bool = False) -> None:
|
||||
print(f"Pre-test cleanup note: {e}")
|
||||
|
||||
|
||||
def run_post_test_cleanup(enable_force: bool = False) -> None:
|
||||
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force:
|
||||
print(_SKIPPED_GPU_CLEANUP_MSG)
|
||||
return
|
||||
|
||||
def run_post_test_cleanup() -> None:
|
||||
if torch.cuda.is_available():
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -318,6 +300,5 @@ __all__ = [
|
||||
"get_physical_device_indices",
|
||||
"run_post_test_cleanup",
|
||||
"run_pre_test_cleanup",
|
||||
"run_forced_gpu_cleanup_round",
|
||||
"wait_for_gpu_memory_to_clear",
|
||||
]
|
||||
@@ -27,10 +27,12 @@ def model_prefix() -> str:
|
||||
return f"{prefix.rstrip('/')}/" if prefix else ""
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@pytest.fixture
|
||||
def clean_gpu_memory_between_tests():
|
||||
# Import here so ``tests.helpers.env`` (and vLLM platform modules) load only
|
||||
# after session autouse fixtures like ``default_env`` have run (RFC #2299).
|
||||
"""Opt-in GPU pre/post hooks for a test (no environment-variable gate).
|
||||
|
||||
Use as a test parameter or ``@pytest.mark.usefixtures("clean_gpu_memory_between_tests")``.
|
||||
"""
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
|
||||
print("\n=== PRE-TEST GPU CLEANUP ===")
|
||||
|
||||
@@ -35,7 +35,7 @@ from tests.helpers.assertions import (
|
||||
assert_diffusion_response,
|
||||
assert_omni_response,
|
||||
)
|
||||
from tests.helpers.env import run_forced_gpu_cleanup_round
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.media import (
|
||||
_merge_base64_audio_to_segment,
|
||||
convert_audio_bytes_to_text,
|
||||
@@ -201,7 +201,8 @@ class OmniServer:
|
||||
env_dict: dict[str, str] | None = None,
|
||||
use_omni: bool = True,
|
||||
) -> None:
|
||||
run_forced_gpu_cleanup_round()
|
||||
run_pre_test_cleanup()
|
||||
run_post_test_cleanup()
|
||||
cleanup_dist_env_and_memory()
|
||||
self.model = model
|
||||
self.serve_args = serve_args
|
||||
@@ -306,7 +307,8 @@ class OmniServer:
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.proc:
|
||||
self._kill_process_tree(self.proc.pid)
|
||||
run_forced_gpu_cleanup_round()
|
||||
run_pre_test_cleanup()
|
||||
run_post_test_cleanup()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@@ -593,7 +595,8 @@ class OmniServerStageCli(OmniServer):
|
||||
proc = self.stage_procs[stage_key]
|
||||
if proc.poll() is None:
|
||||
self._kill_process_tree(proc.pid)
|
||||
run_forced_gpu_cleanup_round()
|
||||
run_pre_test_cleanup()
|
||||
run_post_test_cleanup()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@@ -1272,7 +1275,8 @@ class OmniRunner:
|
||||
**kwargs,
|
||||
) -> None:
|
||||
cleanup_dist_env_and_memory()
|
||||
run_forced_gpu_cleanup_round()
|
||||
run_pre_test_cleanup()
|
||||
run_post_test_cleanup()
|
||||
self.model_name = model_name
|
||||
self.seed = seed
|
||||
self._prompt_len_estimate_cache: dict[str, Any] = {}
|
||||
@@ -1538,7 +1542,8 @@ class OmniRunner:
|
||||
if hasattr(self.omni, "close"):
|
||||
self.omni.close()
|
||||
self._cleanup_process()
|
||||
run_forced_gpu_cleanup_round()
|
||||
run_pre_test_cleanup()
|
||||
run_post_test_cleanup()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@@ -1876,6 +1881,5 @@ __all__ = [
|
||||
"OmniServerStageCli",
|
||||
"OpenAIClientHandler",
|
||||
"get_open_port",
|
||||
"run_forced_gpu_cleanup_round",
|
||||
"dummy_messages_from_mix_data",
|
||||
]
|
||||
Reference in new issue
Block a user