[CI] Restructure vLLM-Omni Test Layout, Fixture Scope, and Support Modules (#2620)

Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-04-20 14:23:42 +08:00
1 parent 0393c58533
commit 8a9add1c81
130 files changed
+4549 -5306

No files matched your search

+1 -1
View File
@@ -552,7 +552,7 @@ steps:
- label: ":full_moon: Diffusion X2V · Accuracy Test"
timeout_in_minutes: 180
commands:
- pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py --run-level advanced_model
- pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m advanced_model --run-level advanced_model
agents:
queue: "mithril-h100-pool"
plugins:
@@ -40,7 +40,7 @@ Currently all the features are available in online serving mode. Hence, only nee
- Test marks: always add `advanced_model` and `diffusion`. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/).
- To maximize code reuse, you may refer to
- `tests/conftest.py` for `omni_server` (running server in subprocess) and `openai_client` fixtures (sending requests and validating output), `generate_synthetic_image` and `assert_XXX_valid` helper.
- `tests/utils.py` for `@hardware_test(...)` and `hardware_marks`.
- `tests/helpers/mark.py` for `@hardware_test(...)` and `hardware_marks`.
- [Parametrizing tests (pytest doc)](https://docs.pytest.org/en/stable/example/parametrize.html) to reuse test function implementation for different cases.
- Doc: add a concise docstring for each test function.
- Reference L4 test implementation: [tests/e2e/online_serving/test_qwen_image_edit_expansion.py](https://github.com/vllm-project/vllm-omni/blob/main/tests/e2e/online_serving/test_qwen_image_edit_expansion.py).
+6 -6
View File
@@ -38,7 +38,7 @@ Defined in `pyproject.toml`:
### Example usage for markers
```python
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
@pytest.mark.core_model
@pytest.mark.omni
@@ -53,7 +53,7 @@ def test_video_to_audio()
### Decorator: `@hardware_test`
This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/utils.py` performs the following actions:
This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
1. **Applies platform and resource markers**
Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
@@ -105,7 +105,7 @@ This decorator is intended to make hardware-aware, cross-platform test authoring
`hardware_marks` returns a list of pytest mark objects with the same signature as `@hardware_test`. Use it when you need more flexibility, such as attaching hardware marks to individual `pytest.param` entries rather than an entire test function.
```python
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
MULTI_CARD_MARKS = hardware_marks(
res={"cuda": "H100", "rocm": "MI325", "npu": "A2"}, num_cards=2
@@ -133,9 +133,9 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
"distributed_tpu: Tests that require multiple TPU devices",
]
```
2. **Implement a marker construction function for your platform** in `vllm-omni/tests/utils.py`:
2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
```python
# In vllm-omni/tests/utils.py
# In vllm-omni/tests/helpers/mark.py
def tpu_marks(*, res: str, num_cards: int):
test_platform = pytest.mark.tpu
@@ -175,4 +175,4 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
- Plug into `hardware_marks`
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
See code in `vllm-omni/tests/utils.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
+6 -8
View File
@@ -221,15 +221,13 @@ from pathlib import Path
import openai
import pytest
from tests.conftest import (
OmniServer,
convert_audio_to_text,
from tests.helpers.media import (
convert_audio_bytes_to_text,
cosine_similarity_text,
dummy_messages_from_mix_data,
generate_synthetic_video,
merge_base64_and_convert_to_text,
)
from tests.utils import get_deploy_config_path
from tests.helpers.runtime import OmniServer, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
# Edit: model name and stage config path
@@ -406,7 +404,7 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
# PURPOSE: Verify text and audio outputs convey the same information
# CUSTOMIZATION: Adjust similarity threshold (0.9) based on accuracy requirements
assert audio_data is not None, "No audio output is generated"
audio_content = merge_base64_and_convert_to_text(audio_data)
audio_content = convert_audio_bytes_to_text(audio_data)
print(f"text content is: {text_content}")
print(f"audio content is: {audio_content}")
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
@@ -429,7 +427,7 @@ from pathlib import Path
import pytest
from vllm.assets.video import VideoAsset
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from ..multi_stages.conftest import OmniRunner
# Optional: set process start method for workers
+61 -3519
View File
File diff suppressed because it is too large. Load diff
+23 -21
View File
@@ -1,5 +1,3 @@
from unittest.mock import Mock, patch
import pytest
import torch
@@ -19,10 +17,14 @@ class MockInputBatch:
self.req_ids = ["req1", "req2"]
self.req_id_to_index = {req_id: i for i, req_id in enumerate(self.req_ids)}
self.num_computed_tokens_cpu = num_computed_tokens_cpu
# Block table is only mocked for validation of length;
# we don't actually need to add valid values here since
# we patch the table when testing.
self.block_table = Mock()
class _DummyBlockTable:
pass
self.block_table = _DummyBlockTable()
self.block_table.block_tables = [None]
@@ -186,7 +188,7 @@ def fake_get_cached_block_ids(self, req_idx, *args, **kwargs):
@pytest.mark.parametrize("num_tokens_padded", [None, 16])
def test_get_merged_hidden_states(num_tokens_padded):
def test_get_merged_hidden_states(num_tokens_padded, mocker):
"""Ensure that hidden states are merged correctly."""
cache = get_omni_pcache()
@@ -221,16 +223,16 @@ def test_get_merged_hidden_states(num_tokens_padded):
input_batch = MockInputBatch(num_computed_tokens_cpu=torch.Tensor([orig_num_tokens_unpadded, 0]))
with patch(
mocker.patch(
"vllm_omni.core.prefix_cache.OmniTensorPrefixCache._get_cached_block_ids",
new=fake_get_cached_block_ids,
):
merged_states = cache.get_merged_hidden_states(
query_start_loc=[0, num_new_toks_req1],
input_batch=input_batch,
hidden_states=new_hidden_states,
num_scheduled_tokens=num_scheduled_tokens,
)
)
merged_states = cache.get_merged_hidden_states(
query_start_loc=[0, num_new_toks_req1],
input_batch=input_batch,
hidden_states=new_hidden_states,
num_scheduled_tokens=num_scheduled_tokens,
)
assert "req1" in merged_states and "req2" in merged_states
req1_merged_states = merged_states["req1"]
@@ -255,7 +257,7 @@ def test_get_merged_hidden_states(num_tokens_padded):
{"foo": 100, "bar": 50, "baz": 10},
],
)
def test_get_merged_multimodal_outputs(feat_dims, num_tokens_padded):
def test_get_merged_multimodal_outputs(feat_dims, num_tokens_padded, mocker):
cache = get_omni_pcache_with_mm_tensors(feat_dims, seq_len=DEFAULT_SEQ_LEN)
orig_num_tokens_unpadded = 8
@@ -298,16 +300,16 @@ def test_get_merged_multimodal_outputs(feat_dims, num_tokens_padded):
input_batch = MockInputBatch(num_computed_tokens_cpu=torch.Tensor([orig_num_tokens_unpadded, 0]))
with patch(
mocker.patch(
"vllm_omni.core.prefix_cache.OmniTensorPrefixCache._get_cached_block_ids",
new=fake_get_cached_block_ids,
):
merged_mm_outputs = cache.get_merged_multimodal_states(
query_start_loc=[0, num_new_toks_req1],
input_batch=input_batch,
multimodal_outputs=new_mm_outputs,
num_scheduled_tokens=num_scheduled_tokens,
)
)
merged_mm_outputs = cache.get_merged_multimodal_states(
query_start_loc=[0, num_new_toks_req1],
input_batch=input_batch,
multimodal_outputs=new_mm_outputs,
num_scheduled_tokens=num_scheduled_tokens,
)
# Ensure the passthrough data wasn't dropped
assert "passthrough_data" in merged_mm_outputs
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import Any
import pytest
from tests.conftest import modify_stage_config
from tests.helpers.stage_config import modify_stage_config
def load_configs(config_path: str) -> list[dict[str, Any]]:
+1 -1
View File
@@ -8,7 +8,6 @@ from typing import Any
import pytest
from tests.conftest import OmniServer
from tests.dfx.conftest import (
create_benchmark_indices,
create_test_parameter_mapping,
@@ -16,6 +15,7 @@ from tests.dfx.conftest import (
get_benchmark_params_for_server,
load_configs,
)
from tests.helpers.runtime import OmniServer
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+11 -119
View File
@@ -1,125 +1,17 @@
"""
Stability-specific conftest: when pytest is executed under this directory,
resource monitoring is started before each test and finalized after each test,
so each stability test case gets its own HTML report (one report per case).
No need to wrap pytest with `bash resource_monitor.sh run -- pytest ...`.
"""
"""Stability pytest hooks and fixtures."""
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
import pytest
STABILITY_DIR = Path(__file__).resolve().parent
RESOURCE_MONITOR_SCRIPT = STABILITY_DIR / "scripts" / "resource_monitor.sh"
REPO_ROOT = STABILITY_DIR.parent.parent.parent
def _start_resource_monitor():
"""Start `resource_monitor.sh start` in the background and return `Popen` or `None`."""
if not RESOURCE_MONITOR_SCRIPT.is_file():
return None
try:
proc = subprocess.Popen(
["bash", str(RESOURCE_MONITOR_SCRIPT), "start", "--backend", "gpu"],
cwd=str(REPO_ROOT),
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
try:
proc.wait(timeout=2)
if proc.returncode != 0:
stderr = proc.stderr.read().decode("utf-8", errors="ignore") if proc.stderr else ""
if stderr.strip():
sys.stderr.write(f"[Stability] Resource monitor failed to start: {stderr.strip()}\n")
return None
except subprocess.TimeoutExpired:
pass
return proc
except (FileNotFoundError, OSError):
return None
def _get_monitor_data_root() -> Path:
data_root = os.environ.get("RESOURCE_MONITOR_DATA_ROOT") or os.environ.get("GPU_MONITOR_DATA_ROOT")
if data_root:
return Path(data_root)
return STABILITY_DIR / "gpu_monitor_data"
def _wait_for_run_dir(timeout_sec: int = 10) -> Path | None:
data_root = _get_monitor_data_root()
run_id_file = data_root / "current_run_id"
deadline = time.time() + timeout_sec
while time.time() < deadline:
if run_id_file.is_file():
run_id = run_id_file.read_text(encoding="utf-8").strip()
if run_id:
run_dir = data_root / run_id
if run_dir.is_dir():
return run_dir
time.sleep(0.5)
return None
def _report_latest_gpu_samples(stop_event: threading.Event) -> None:
"""Periodically print the latest sampled GPU line."""
log_interval = int(
os.environ.get("RESOURCE_MONITOR_LOG_INTERVAL") or os.environ.get("GPU_MONITOR_LOG_INTERVAL") or "15"
)
log_interval = max(log_interval, 1)
last_line = ""
time.sleep(min(log_interval, 5))
while not stop_event.wait(log_interval):
run_dir = _wait_for_run_dir(timeout_sec=1)
if run_dir is None:
continue
csv_file = run_dir / "gpu_metrics.csv"
if not csv_file.is_file():
continue
try:
lines = csv_file.read_text(encoding="utf-8").splitlines()
except OSError:
continue
if len(lines) <= 1:
continue
latest = lines[-1].strip()
if latest and latest != last_line:
last_line = latest
sys.stderr.write(f"[GPU] {latest}\n")
def _finalize_resource_monitor() -> str | None:
"""
Run `resource_monitor.sh finalize` for the current run and generate the report.
Returns the bundle dir path (for this test case's report) if successful, else None.
"""
if not RESOURCE_MONITOR_SCRIPT.is_file():
return None
try:
result = subprocess.run(
["bash", str(RESOURCE_MONITOR_SCRIPT), "finalize", "--backend", "gpu"],
cwd=str(REPO_ROOT),
capture_output=True,
text=True,
timeout=60,
check=False,
)
if result.returncode != 0:
return None
for line in (result.stdout or "").splitlines():
if line.startswith("GPU_MONITOR_BUNDLE_DIR=") or line.startswith("RESOURCE_MONITOR_BUNDLE_DIR="):
_, _, value = line.partition("=")
return value.strip() if value else None
return None
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return None
from tests.dfx.stability.helpers import (
finalize_resource_monitor,
report_latest_gpu_samples,
start_resource_monitor,
wait_for_run_dir,
)
@pytest.fixture(autouse=True)
@@ -128,19 +20,19 @@ def stability_resource_monitor_per_test(request: pytest.FixtureRequest):
For each test under this directory: start GPU monitor before the test,
then finalize after the test so this case gets its own report.html.
"""
proc = _start_resource_monitor()
proc = start_resource_monitor()
stop_event = threading.Event()
reporter: threading.Thread | None = None
if proc is not None:
reporter = threading.Thread(
target=_report_latest_gpu_samples,
target=report_latest_gpu_samples,
args=(stop_event,),
name="stability-resource-monitor-reporter",
daemon=True,
)
reporter.start()
run_dir = _wait_for_run_dir(timeout_sec=5)
run_dir = wait_for_run_dir(timeout_sec=5)
node_name = request.node.name
if run_dir is not None:
sys.stderr.write(f"[Stability] Resource monitor started for test: {node_name} | run dir: {run_dir}\n")
@@ -161,7 +53,7 @@ def stability_resource_monitor_per_test(request: pytest.FixtureRequest):
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
bundle_dir = _finalize_resource_monitor()
bundle_dir = finalize_resource_monitor()
node_name = request.node.name
if bundle_dir:
sys.stderr.write(f"[Stability] Report for test «{node_name}»: {bundle_dir}/report.html\n")
+117
View File
@@ -0,0 +1,117 @@
"""Stability resource monitor helpers."""
from __future__ import annotations
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
STABILITY_DIR = Path(__file__).resolve().parent
RESOURCE_MONITOR_SCRIPT = STABILITY_DIR / "scripts" / "resource_monitor.sh"
REPO_ROOT = STABILITY_DIR.parent.parent.parent
def start_resource_monitor():
"""Start `resource_monitor.sh start` in the background and return `Popen` or `None`."""
if not RESOURCE_MONITOR_SCRIPT.is_file():
return None
try:
proc = subprocess.Popen(
["bash", str(RESOURCE_MONITOR_SCRIPT), "start", "--backend", "gpu"],
cwd=str(REPO_ROOT),
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
try:
proc.wait(timeout=2)
if proc.returncode != 0:
stderr = proc.stderr.read().decode("utf-8", errors="ignore") if proc.stderr else ""
if stderr.strip():
sys.stderr.write(f"[Stability] Resource monitor failed to start: {stderr.strip()}\n")
return None
except subprocess.TimeoutExpired:
pass
return proc
except (FileNotFoundError, OSError):
return None
def get_monitor_data_root() -> Path:
data_root = os.environ.get("RESOURCE_MONITOR_DATA_ROOT") or os.environ.get("GPU_MONITOR_DATA_ROOT")
if data_root:
return Path(data_root)
return STABILITY_DIR / "gpu_monitor_data"
def wait_for_run_dir(timeout_sec: int = 10) -> Path | None:
data_root = get_monitor_data_root()
run_id_file = data_root / "current_run_id"
deadline = time.time() + timeout_sec
while time.time() < deadline:
if run_id_file.is_file():
run_id = run_id_file.read_text(encoding="utf-8").strip()
if run_id:
run_dir = data_root / run_id
if run_dir.is_dir():
return run_dir
time.sleep(0.5)
return None
def report_latest_gpu_samples(stop_event: threading.Event) -> None:
"""Periodically print the latest sampled GPU line."""
log_interval = int(
os.environ.get("RESOURCE_MONITOR_LOG_INTERVAL") or os.environ.get("GPU_MONITOR_LOG_INTERVAL") or "15"
)
log_interval = max(log_interval, 1)
last_line = ""
time.sleep(min(log_interval, 5))
while not stop_event.wait(log_interval):
run_dir = wait_for_run_dir(timeout_sec=1)
if run_dir is None:
continue
csv_file = run_dir / "gpu_metrics.csv"
if not csv_file.is_file():
continue
try:
lines = csv_file.read_text(encoding="utf-8").splitlines()
except OSError:
continue
if len(lines) <= 1:
continue
latest = lines[-1].strip()
if latest and latest != last_line:
last_line = latest
sys.stderr.write(f"[GPU] {latest}\n")
def finalize_resource_monitor() -> str | None:
"""
Run `resource_monitor.sh finalize` for the current run and generate the report.
Returns the bundle dir path (for this test case's report) if successful, else None.
"""
if not RESOURCE_MONITOR_SCRIPT.is_file():
return None
try:
result = subprocess.run(
["bash", str(RESOURCE_MONITOR_SCRIPT), "finalize", "--backend", "gpu"],
cwd=str(REPO_ROOT),
capture_output=True,
text=True,
timeout=60,
check=False,
)
if result.returncode != 0:
return None
for line in (result.stdout or "").splitlines():
if line.startswith("GPU_MONITOR_BUNDLE_DIR=") or line.startswith("RESOURCE_MONITOR_BUNDLE_DIR="):
_, _, value = line.partition("=")
return value.strip() if value else None
return None
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return None
@@ -24,7 +24,6 @@ from typing import Any
import pytest
from tests.conftest import OmniServer
from tests.dfx.conftest import (
create_benchmark_indices,
create_test_parameter_mapping,
@@ -33,6 +32,7 @@ from tests.dfx.conftest import (
load_configs,
)
from tests.dfx.perf.scripts.run_benchmark import run_benchmark
from tests.helpers.runtime import OmniServer
STABILITY_DIR = Path(__file__).resolve().parent.parent
DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
+1 -1
View File
@@ -21,7 +21,7 @@ from unittest.mock import MagicMock, Mock, patch
import pytest
import torch
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.cache.teacache.extractors import extract_flux2_context, extract_flux2_klein_context
from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import (
Flux2Transformer2DModel,
@@ -17,7 +17,7 @@ import pytest
import torch
import torch.distributed as dist
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.attention.parallel.ulysses import (
_all_gather_int,
_ulysses_all_to_all_any_o,
File renamed without changes.
+1 -1
View File
@@ -8,7 +8,7 @@ import torch
from vllm.lora.lora_weights import LoRALayerWeights
from vllm.lora.utils import get_supported_lora_modules
from tests.diffusion.lora.conftest import (
from tests.diffusion.lora.helpers import (
DummyBaseLayerWithLoRA,
FakeLinearBase,
fake_replace_submodule,
@@ -11,7 +11,7 @@ import pytest
import torch
from safetensors.torch import save_file
from tests.diffusion.lora.conftest import (
from tests.diffusion.lora.helpers import (
DummyBaseLayerWithLoRA,
FakeLinearBase,
fake_replace_submodule,
@@ -2,7 +2,7 @@ import pytest
import torch
from pytest_mock import MockerFixture
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.models.flux2.flux2_transformer import (
Flux2PosEmbed,
Flux2Transformer2DModel,
@@ -2,27 +2,26 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GLM-Image Sequence Parallelism support."""
from unittest.mock import MagicMock, patch
import pytest
from vllm_omni.diffusion.data import DiffusionParallelConfig
@pytest.fixture(scope="function", autouse=True)
def setup_sp_groups():
def setup_sp_groups(mocker):
"""Set up SP and TP groups for each test function."""
with patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group") as mock_get_sp_group:
with patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1):
with patch("vllm.distributed.parallel_state.get_tp_group") as mock_get_tp_group:
mock_sp_group = MagicMock()
mock_sp_group.world_size = 4
mock_get_sp_group.return_value = mock_sp_group
mock_get_sp_group = mocker.patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group")
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
mock_get_tp_group = mocker.patch("vllm.distributed.parallel_state.get_tp_group")
mock_tp_group = MagicMock()
mock_tp_group.world_size = 1
mock_get_tp_group.return_value = mock_tp_group
yield
mock_sp_group = mocker.MagicMock()
mock_sp_group.world_size = 4
mock_get_sp_group.return_value = mock_sp_group
mock_tp_group = mocker.MagicMock()
mock_tp_group.world_size = 1
mock_get_tp_group.return_value = mock_tp_group
yield
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -32,7 +32,7 @@ import numpy as np
import pytest
import torch
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
# ---------------------------------------------------------------------------
# Configuration — add new quantization methods / models here
@@ -8,7 +8,7 @@ import pytest
import torch
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
pytestmark = [pytest.mark.diffusion]
@@ -13,7 +13,7 @@ import torch
from pytest_mock import MockerFixture
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+1 -14
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import os
import shutil
import subprocess
from contextlib import contextmanager
from dataclasses import dataclass
@@ -13,7 +12,7 @@ import requests
import torch
from PIL import Image
from tests.conftest import OmniServer, OmniServerParams
from tests.helpers.runtime import OmniServer, OmniServerParams
def pytest_addoption(parser):
@@ -208,18 +207,6 @@ def rabbit_image(accuracy_artifact_root: Path) -> Image.Image:
return image
def reset_artifact_dir(path: Path) -> Path:
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
return path
def infer_model_label(model: str) -> str:
label = Path(model.rstrip("/\\")).name or "model"
return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in label)
def _build_accuracy_server_config(
*,
generate_model: str,
@@ -1,5 +1,3 @@
from __future__ import annotations
from pathlib import Path
import numpy as np
@@ -9,6 +7,20 @@ from PIL import Image
from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure
def reset_artifact_dir(path: Path) -> Path:
import shutil
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
return path
def infer_model_label(model: str) -> str:
label = Path(model.rstrip("/\\")).name or "model"
return "".join(char if char.isalnum() or char in {"-", "_"} else "_" for char in label)
def model_output_dir(parent_dir: Path, model: str) -> Path:
safe_model_name = model.split("/")[-1].replace(".", "_")
path = parent_dir / safe_model_name
@@ -6,8 +6,8 @@ from pathlib import Path
import pytest
from benchmarks.accuracy.text_to_image.gbench import main as gbench_main
from tests.e2e.accuracy.conftest import infer_model_label, reset_artifact_dir
from tests.utils import hardware_test
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
@pytest.mark.advanced_model
@@ -7,8 +7,8 @@ import pytest
from benchmarks.accuracy.image_to_image.gedit_bench import GROUPS
from benchmarks.accuracy.image_to_image.gedit_bench import main as gedit_main
from tests.e2e.accuracy.conftest import infer_model_label, reset_artifact_dir
from tests.utils import hardware_test
from tests.e2e.accuracy.helpers import infer_model_label, reset_artifact_dir
from tests.helpers.mark import hardware_test
@pytest.mark.advanced_model
+6 -9
View File
@@ -12,13 +12,10 @@ import torch
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from PIL import Image
from tests.conftest import (
OmniServer,
_run_post_test_cleanup,
_run_pre_test_cleanup,
)
from tests.e2e.accuracy.utils import assert_similarity, model_output_dir
from tests.utils import hardware_test
from tests.e2e.accuracy.helpers import assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
MODEL_ID = "Qwen/Qwen-Image"
MODEL_ENV_VAR = "QWEN_IMAGE_MODEL"
@@ -70,7 +67,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image:
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
_run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup(enable_force=True)
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -99,7 +96,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
_run_post_test_cleanup(enable_force=True)
run_post_test_cleanup(enable_force=True)
@pytest.mark.advanced_model
+6 -9
View File
@@ -10,13 +10,10 @@ from diffusers import QwenImageEditPipeline, QwenImageEditPlusPipeline
from PIL import Image
from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
from tests.conftest import (
OmniServer,
_run_post_test_cleanup,
_run_pre_test_cleanup,
)
from tests.e2e.accuracy.utils import assert_similarity, model_output_dir
from tests.utils import hardware_test
from tests.e2e.accuracy.helpers import assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
SINGLE_MODEL = "Qwen/Qwen-Image-Edit"
MULTIPLE_MODEL = "Qwen/Qwen-Image-Edit-2509"
@@ -77,7 +74,7 @@ def _run_diffusers_image_edit(
input_images: list[Image.Image],
output_path: Path,
) -> Image.Image:
_run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup(enable_force=True)
pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None
device = torch.device("cuda:0")
torch.cuda.set_device(device)
@@ -110,7 +107,7 @@ def _run_diffusers_image_edit(
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
_run_post_test_cleanup(enable_force=True)
run_post_test_cleanup(enable_force=True)
def _vllm_omni_output_single_image(
@@ -12,13 +12,10 @@ import torch
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from PIL import Image
from tests.conftest import (
OmniServer,
_run_post_test_cleanup,
_run_pre_test_cleanup,
)
from tests.e2e.accuracy.utils import assert_image_sequence_similarity, model_output_dir
from tests.utils import hardware_test
from tests.e2e.accuracy.helpers import assert_image_sequence_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
MODEL_ID = "Qwen/Qwen-Image-Layered"
MODEL_ENV_VAR = "QWEN_IMAGE_LAYERED_MODEL"
@@ -93,7 +90,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o
def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]:
_run_pre_test_cleanup(enable_force=True)
run_pre_test_cleanup(enable_force=True)
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -126,7 +123,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
_run_post_test_cleanup(enable_force=True)
run_post_test_cleanup(enable_force=True)
@pytest.mark.advanced_model
@@ -22,7 +22,6 @@ import torch
from diffusers import UniPCMultistepScheduler
from PIL import Image
from tests.conftest import OmniServerParams
from tests.e2e.accuracy.wan22_i2v.run_wan22_i2v_diffusers_cp import (
_configure_scheduler,
_ensure_wan_ftfy_fallback,
@@ -48,7 +47,8 @@ from tests.e2e.accuracy.wan22_i2v.wan22_i2v_video_similarity_common import (
SSIM_THRESHOLD,
WIDTH,
)
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
def test_parse_video_metadata_extracts_dimensions_and_fps() -> None:
@@ -26,7 +26,7 @@ from contextlib import ExitStack
import pytest
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -19,7 +19,7 @@ import numpy as np
import pytest
from transformers import AutoTokenizer
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -10,7 +10,7 @@ import pytest
from tests.e2e.offline_inference.custom_pipeline.worker_extension import (
vLLMOmniColocateWorkerExtensionForTest,
)
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
from vllm_omni.entrypoints.async_omni import AsyncOmni
@@ -22,9 +22,10 @@ import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
# Reference pixel data extracted from the known-good output image
@@ -31,9 +31,10 @@ import torch
from PIL import Image
from safetensors.torch import save_file
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
@@ -27,9 +27,10 @@ from typing import Any
import pytest
from PIL import Image
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
# Reference pixel data extracted from the known-good output image
@@ -26,8 +26,9 @@ from pathlib import Path
import pytest
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "bagel_sharedmemory_ci.yaml")
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -26,8 +26,8 @@ import yaml
from huggingface_hub import snapshot_download
from vllm.sampling_params import SamplingParams
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
@@ -4,8 +4,9 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -2,8 +2,8 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -7,7 +7,7 @@ import pytest
import torch
from safetensors.torch import save_file
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -18,7 +18,7 @@ import pytest
import torch
from transformers import AutoTokenizer
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -18,8 +18,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -14,8 +14,9 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -13,7 +13,7 @@ import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -8,7 +8,7 @@ import torch.nn.functional as F
from PIL import Image
from transformers import CLIPModel, CLIPProcessor
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -11,7 +11,7 @@ import numpy as np
import pytest
from PIL import Image
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
REPO_ROOT = Path(__file__).resolve().parents[3]
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
@@ -8,8 +8,8 @@ import av
import numpy as np
import pytest
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -23,8 +23,10 @@ import pytest
import torch
from vllm.sampling_params import SamplingParams
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
# ---------------------------------------------------------------------------
# Constants
@@ -10,13 +10,13 @@ from pathlib import Path
import pytest
from tests.conftest import (
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.helpers.stage_config import modify_stage_config
models = ["Jonathan1909/Ming-flash-omni-2.0"]
@@ -16,8 +16,8 @@ from pathlib import Path
import numpy as np
import pytest
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
MODEL = "k2-fsa/OmniVoice"
@@ -16,7 +16,7 @@ import pytest
import torch
from pytest_mock import MockerFixture
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
@@ -36,8 +36,8 @@ from typing import Any
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -4,13 +4,13 @@ E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text outp
import pytest
from tests.conftest import (
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen2.5-Omni-7B"]
@@ -9,11 +9,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_video
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
@@ -15,8 +15,8 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import modify_stage_config
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
@@ -15,8 +15,8 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import modify_stage_config
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
@@ -36,8 +36,8 @@ from pathlib import Path
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -20,8 +20,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -15,8 +15,8 @@ import numpy as np
import pytest
import torch
from tests.conftest import assert_audio_valid
from tests.utils import hardware_test
from tests.helpers.assertions import assert_audio_valid
from tests.helpers.mark import hardware_test
from vllm_omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -1,7 +1,7 @@
import pytest
import torch
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -3,7 +3,7 @@ import os
import pytest
import torch
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
+2 -2
View File
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -18,7 +18,7 @@ if str(REPO_ROOT) not in sys.path:
import time
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.platforms import current_omni_platform
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
+5 -5
View File
@@ -12,9 +12,9 @@ import numpy as np
import pytest
import torch
import tests.conftest as omni_test_conftest
from tests.conftest import OmniRunner
from tests.utils import hardware_test
import tests.helpers.runtime as omni_runtime
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
prepare_voxcpm_hf_config_dir,
resolve_voxcpm_model_dir,
@@ -30,7 +30,7 @@ SAMPLE_RATE = 24000
@pytest.fixture(autouse=True)
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
original_cleanup = omni_test_conftest.cleanup_dist_env_and_memory
original_cleanup = omni_runtime.cleanup_dist_env_and_memory
def _safe_cleanup() -> None:
try:
@@ -40,7 +40,7 @@ def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
return
raise
monkeypatch.setattr(omni_test_conftest, "cleanup_dist_env_and_memory", _safe_cleanup)
monkeypatch.setattr(omni_runtime, "cleanup_dist_env_and_memory", _safe_cleanup)
def _build_prompt(text: str) -> dict[str, Any]:
+2 -2
View File
@@ -5,8 +5,8 @@ import os
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
VOXCPM2_MODEL = "openbmb/VoxCPM2"
STAGE_CONFIG = os.path.join(
@@ -29,8 +29,9 @@ from mistral_common.protocol.speech.request import SpeechRequest
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
from vllm import SamplingParams
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.async_omni import AsyncOmni
MODEL = "mistralai/Voxtral-4B-TTS-2603"
@@ -20,8 +20,9 @@ import pytest
import torch
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -213,7 +214,10 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
)
@pytest.mark.integration
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu():
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.")
@@ -16,13 +16,8 @@ assert_diffusion_response validates successful generation and the expected
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
PROMPT = "A futuristic city skyline at twilight, cyberpunk style, ultra-detailed, high resolution."
NEGATIVE_PROMPT = "low quality, blurry, distorted, deformed, watermark"
@@ -138,7 +133,7 @@ def test_bagel(
- Ulysses-SP (degree=2)
- Ring-Attention (degree=2)
Validation is delegated to assert_diffusion_response in tests.conftest,
Validation is delegated to assert_diffusion_response in tests/helpers/assertions.py,
which checks output dimensions and basic correctness.
"""
@@ -28,8 +28,8 @@ from pathlib import Path
import pytest
from vllm.assets.image import ImageAsset
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -16,8 +16,8 @@ from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
@@ -15,9 +15,9 @@ import pytest
import soundfile as sf
from vllm.assets.image import ImageAsset
from tests import conftest as tests_conftest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import convert_audio_bytes_to_text
from tests.helpers.runtime import OmniServerParams
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -87,7 +87,7 @@ def _convert_audio_bytes_to_text_without_ffmpeg(raw_bytes: bytes) -> str:
@pytest.fixture
def dynin_t2s_openai_client(openai_client, monkeypatch):
monkeypatch.setattr(
tests_conftest,
convert_audio_bytes_to_text,
"convert_audio_bytes_to_text",
_convert_audio_bytes_to_text_without_ffmpeg,
)
@@ -11,12 +11,8 @@ Coverage:
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
POSITIVE_PROMPT = "A cat sitting on a windowsill"
@@ -14,7 +14,7 @@ import httpx
import pytest
from PIL import Image, ImageDraw
from tests.conftest import OmniServer, OmniServerParams
from tests.helpers.runtime import OmniServer, OmniServerParams
MODEL = "black-forest-labs/FLUX.2-klein-4B"
@@ -14,13 +14,8 @@ image resolution.
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
MODEL = "black-forest-labs/FLUX.2-dev"
PROMPT = "A cinematic mountain landscape at sunrise, dramatic clouds, ultra-detailed, realistic photography."
@@ -5,13 +5,8 @@ and are supported by the FluxKontext model.
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
generate_synthetic_image,
)
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
@@ -11,12 +11,8 @@ Coverage (H100, since model cannot fit L4):
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion."
NEGATIVE_PROMPT = "low quality, blurry, distorted"
@@ -22,7 +22,7 @@ from PIL import Image
from vllm.assets.image import ImageAsset
from vllm.utils.network_utils import get_open_port
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
# Increase timeout for downloading assets from S3 (default 5s is too short for CI)
@@ -22,8 +22,8 @@ import torch
from PIL import Image
from safetensors.torch import save_file
from tests.conftest import OmniServer
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -13,14 +13,9 @@ This validates:
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
generate_synthetic_image,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting."
NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
@@ -13,13 +13,8 @@ This validates:
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
TEXT_TO_IMAGE_PROMPT = (
"A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed."
+4 -7
View File
@@ -9,13 +9,10 @@ from pathlib import Path
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -10,15 +10,14 @@ from pathlib import Path
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import modify_stage_config
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -6,13 +6,13 @@ import os
import pytest
from tests.conftest import (
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
from tests.utils import hardware_marks
# L4: 4 GPUs + TP=4; XPU B60: 2 cards (use num_cards={"cuda": 4, "xpu": 4} if needed)
FOUR_CARD_MARKS = hardware_marks(
+3 -2
View File
@@ -17,8 +17,9 @@ from pathlib import Path
import httpx
import pytest
from tests.conftest import OmniServerParams, generate_synthetic_audio
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio
from tests.helpers.runtime import OmniServerParams
try:
from transformers import HiggsAudioV2TokenizerModel # noqa: F401
@@ -6,15 +6,10 @@ import os
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+4 -9
View File
@@ -6,15 +6,10 @@ import os
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -10,15 +10,10 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
@@ -455,7 +450,7 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
"key_words": {"text": ["london"]},
}
# Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/conftest.py).
# Retry only when assert_omni_response fails on text/audio cosine similarity (see tests/helpers/assertions.py).
_similarity_assert_msg = "The audio content is not same as the text"
_max_retries = 3
for attempt in range(_max_retries):
@@ -520,7 +515,7 @@ def test_speaker_002(omni_server, openai_client) -> None:
"key_words": {"text": ["beijing"]},
}
# Retry only when assert_omni_response fails on preset voice gender (see tests/conftest.py).
# Retry only when assert_omni_response fails on preset voice gender (see tests/helpers/assertions.py).
_gender_assert_substr = "estimated gender"
_max_retries = 3
for attempt in range(_max_retries):
@@ -16,14 +16,14 @@ import wave
import pytest
import websockets
from tests.conftest import (
OmniServerParams,
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
convert_audio_bytes_to_text,
cosine_similarity_text,
generate_synthetic_audio,
modify_stage_config,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -14,8 +14,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import OmniServerParams
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -14,8 +14,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import OmniServerParams
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
@@ -22,12 +22,10 @@ import httpx
import pytest
import yaml
from tests.conftest import (
OmniServer,
convert_audio_file_to_text,
cosine_similarity_text,
)
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_text
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
STAGE_INIT_TIMEOUT_S = 120
@@ -14,8 +14,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import OmniServerParams
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
@@ -14,8 +14,9 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.conftest import OmniServerParams
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
@@ -17,8 +17,9 @@ import struct
import httpx
import pytest
from tests.conftest import OmniServer
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
@@ -11,8 +11,9 @@ import os
import pytest
import websockets
from tests.conftest import OmniServer
from tests.utils import get_deploy_config_path, hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -7,14 +7,9 @@ and are supported by the following models:
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
generate_synthetic_image,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
@@ -12,13 +12,8 @@ See docs/user_guide/diffusion_acceleration.md.
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
NEGATIVE_PROMPT = "blurry, low quality"
@@ -14,15 +14,9 @@ Total distinct features covered: cache_dit, Ulysses-SP, CFG-Parallel, layerwise-
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
decode_b64_image,
dummy_messages_from_mix_data,
generate_synthetic_image,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.media import decode_b64_image, generate_synthetic_image
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
MODEL = "Qwen/Qwen-Image-Layered"
EDIT_PROMPT = "Decompose this image into layers."
@@ -4,12 +4,8 @@ Tests for Stable Diffusion 3.5 medium model.
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
)
from tests.utils import hardware_marks
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
POSITIVE_PROMPT = "A serene mountain landscape at sunset"
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff. Show more