diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index be528b316..6f3ad6504 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -16,11 +16,10 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Voxtral TTS CUDA Unit Test" - timeout_in_minutes: 10 + - label: "CUDA Unit Test with single card" depends_on: upload-ready-pipeline commands: - - "timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py" + - timeout 10m pytest -v -s -m 'core_model and cuda and L4 and not distributed_cuda' --ignore=tests/e2e --ignore=tests/engine/test_async_omni_engine_abort.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml agents: queue: "gpu_1_queue" plugins: @@ -33,6 +32,22 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "CUDA Unit Test with multi cards" + depends_on: upload-ready-pipeline + commands: + - timeout 10m pytest -v -s -m 'core_model and cuda and L4 and distributed_cuda' --ignore=tests/e2e --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml + agents: + queue: "gpu_4_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Model Test" depends_on: upload-ready-pipeline commands: @@ -152,24 +167,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Diffusion GPU Worker Test" - depends_on: upload-ready-pipeline - commands: - - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py - agents: - queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - shm-size: "8gb" - environment: - - "HF_HOME=/fsx/hf_cache" - - "HF_TOKEN" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Engine Test" depends_on: upload-ready-pipeline diff --git a/tests/core/sched/test_generation_scheduler_restore.py b/tests/core/sched/test_generation_scheduler_restore.py index 0eae3c4db..154f40b39 100644 --- a/tests/core/sched/test_generation_scheduler_restore.py +++ b/tests/core/sched/test_generation_scheduler_restore.py @@ -9,6 +9,10 @@ those requests are permanently orphaned. import unittest from collections import deque +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class FakeAdapter: """Minimal mock of OmniChunkTransferAdapter tracking restore calls.""" diff --git a/tests/diffusion/cache/test_teacache_extractors.py b/tests/diffusion/cache/test_teacache_extractors.py index 5ba52ddfe..a52e11b3d 100644 --- a/tests/diffusion/cache/test_teacache_extractors.py +++ b/tests/diffusion/cache/test_teacache_extractors.py @@ -21,12 +21,13 @@ from unittest.mock import MagicMock, Mock, patch import pytest import torch +from tests.utils import hardware_test from vllm_omni.diffusion.cache.teacache.extractors import extract_flux2_klein_context from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import ( Flux2Transformer2DModel, ) -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +pytestmark = [pytest.mark.core_model] @pytest.fixture(scope="function", autouse=True) @@ -113,6 +114,7 @@ class TestFlux2KleinExtractor(BaseExtractorTest): def get_sample_inputs(self, sample_inputs): return sample_inputs + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_modulated_input_shape(self, flux2_klein_module, sample_inputs): """Test that modulated_input has correct shape matching the model's inner_dim. @@ -126,16 +128,19 @@ class TestFlux2KleinExtractor(BaseExtractorTest): inner_dim = flux2_klein_module.inner_dim assert context.modulated_input.shape == (batch_size, img_seq_len, inner_dim) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_run_transformer_blocks_callable(self, flux2_klein_module, sample_inputs): """Test that run_transformer_blocks is callable.""" context = extract_flux2_klein_context(flux2_klein_module, **sample_inputs) assert callable(context.run_transformer_blocks) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_postprocess_callable(self, flux2_klein_module, sample_inputs): """Test that postprocess is callable.""" context = extract_flux2_klein_context(flux2_klein_module, **sample_inputs) assert callable(context.postprocess) + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_extra_states_contains_full_transformer(self, flux2_klein_module, sample_inputs): """Test that extra_states contains run_flux2_full_transformer_with_single.""" context = extract_flux2_klein_context(flux2_klein_module, **sample_inputs) @@ -154,6 +159,7 @@ class TestFlux2KleinExtractor(BaseExtractorTest): assert context is not None assert context.temb is not None + @pytest.mark.cpu def test_invalid_module_raises_error(self): """Test that invalid module without transformer_blocks raises ValueError.""" invalid_module = Mock() diff --git a/tests/diffusion/distributed/test_distributed_vae_executor.py b/tests/diffusion/distributed/test_distributed_vae_executor.py index 93cf3d195..dc491dcda 100644 --- a/tests/diffusion/distributed/test_distributed_vae_executor.py +++ b/tests/diffusion/distributed/test_distributed_vae_executor.py @@ -11,6 +11,8 @@ from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor impor TileTask, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class E2EOperator: """tiles with (2, 3) -- (H,W)""" diff --git a/tests/diffusion/distributed/test_ulysses_uaa_perf.py b/tests/diffusion/distributed/test_ulysses_uaa_perf.py index c8b07ba15..04bbf5ee8 100644 --- a/tests/diffusion/distributed/test_ulysses_uaa_perf.py +++ b/tests/diffusion/distributed/test_ulysses_uaa_perf.py @@ -17,6 +17,7 @@ import pytest import torch import torch.distributed as dist +from tests.utils import hardware_test from vllm_omni.diffusion.attention.parallel.ulysses import ( _all_gather_int, _ulysses_all_to_all_any_o, @@ -69,6 +70,8 @@ PERF_CASES: list[_PerfCase] = [ @pytest.mark.parametrize("case", PERF_CASES) +@pytest.mark.core_model +@hardware_test(res={"cuda": "L4"}, num_cards=4) def test_ulysses_advanced_uaa_comm_overhead(case: _PerfCase) -> None: available_gpus = current_omni_platform.get_device_count() if available_gpus < case.world_size: diff --git a/tests/diffusion/models/flux2/test_flux2_transformer_tp.py b/tests/diffusion/models/flux2/test_flux2_transformer_tp.py index a2d1fe6ab..faad08afd 100644 --- a/tests/diffusion/models/flux2/test_flux2_transformer_tp.py +++ b/tests/diffusion/models/flux2/test_flux2_transformer_tp.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch import pytest import torch +from tests.utils import hardware_test from vllm_omni.diffusion.models.flux2.flux2_transformer import ( Flux2PosEmbed, Flux2Transformer2DModel, @@ -24,6 +25,8 @@ def setup_tp_group(): class TestFlux2TransformerWeightLoading: """Test Flux2Transformer weight loading functionality""" + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_weight_loading_tp2(self, setup_tp_group): """Verify weights load correctly with TP=2""" # Prepare test data @@ -78,6 +81,8 @@ class TestFlux2TransformerWeightLoading: class TestFlux2RopePositionEmbedding: """Test Flux2 RoPE position embedding functionality""" + @pytest.mark.core_model + @pytest.mark.cpu def test_rope_position_embedding(self): """Verify RoPE produces correct embeddings for 4D coordinates""" # Prepare test data - use model default configuration @@ -132,6 +137,8 @@ class TestFlux2RopePositionEmbedding: class TestFlux2PackedModuleMapping: """Test Flux2 packed module mapping functionality""" + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_packed_module_mapping(self, setup_tp_group): """Verify to_qkv packing matches HF checkpoint""" model = Flux2Transformer2DModel( @@ -208,6 +215,8 @@ class TestFlux2PackedModuleMapping: f"add_kv_proj weight dimension should be {expected_add_kv_shape}, got {attn_block.add_kv_proj.weight.shape}" ) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_packed_mapping_edge_cases(self, setup_tp_group): """Test edge cases for packed mapping""" model = Flux2Transformer2DModel( diff --git a/tests/diffusion/quantization/test_fp8_config.py b/tests/diffusion/quantization/test_fp8_config.py index 9c18c1f55..574af7a66 100644 --- a/tests/diffusion/quantization/test_fp8_config.py +++ b/tests/diffusion/quantization/test_fp8_config.py @@ -5,7 +5,7 @@ import pytest from torch import nn -pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] def test_build_quant_config_fp8(): diff --git a/tests/diffusion/test_diffusion_model_runner.py b/tests/diffusion/test_diffusion_model_runner.py index 88b17147e..8768986f0 100644 --- a/tests/diffusion/test_diffusion_model_runner.py +++ b/tests/diffusion/test_diffusion_model_runner.py @@ -8,9 +8,10 @@ import pytest import torch import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module +from tests.utils import hardware_test from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner -pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] +pytestmark = [pytest.mark.diffusion] @contextmanager @@ -64,6 +65,8 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar return runner +@pytest.mark.core_model +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch): """Guard cache diagnostics with runtime backend state to avoid stale-config crashes.""" runner = _make_runner(cache_backend=None, cache_backend_name="cache_dit") @@ -84,6 +87,8 @@ def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypa assert cache_summary_calls == [] +@pytest.mark.core_model +@hardware_test(res={"cuda": "L4"}, num_cards=1) def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeypatch): class _EnabledCacheBackend: def is_enabled(self): @@ -107,6 +112,8 @@ def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeyp assert cache_summary_calls == [(runner.pipeline, True)] +@pytest.mark.core_model +@pytest.mark.cpu def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch): class _DummyLoader: def __init__(self, load_config, od_config=None): diff --git a/tests/distributed/omni_connectors/test_basic_connectors.py b/tests/distributed/omni_connectors/test_basic_connectors.py index bca96e790..662d41fe0 100644 --- a/tests/distributed/omni_connectors/test_basic_connectors.py +++ b/tests/distributed/omni_connectors/test_basic_connectors.py @@ -9,7 +9,7 @@ from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer -# pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def test_basic_serialization(): diff --git a/tests/distributed/omni_coordinator/test_load_balancer.py b/tests/distributed/omni_coordinator/test_load_balancer.py index c54d24894..b2d1f3ee8 100644 --- a/tests/distributed/omni_coordinator/test_load_balancer.py +++ b/tests/distributed/omni_coordinator/test_load_balancer.py @@ -3,12 +3,16 @@ from time import time +import pytest + from vllm_omni.distributed.omni_coordinator import ( InstanceInfo, RandomBalancer, StageStatus, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_load_balancer_select_returns_valid_index(): """Verify RandomBalancer.select() returns a valid index for instances.""" diff --git a/tests/distributed/omni_coordinator/test_omni_coord_client_for_hub.py b/tests/distributed/omni_coordinator/test_omni_coord_client_for_hub.py index 24b331923..2fbd7c85b 100644 --- a/tests/distributed/omni_coordinator/test_omni_coord_client_for_hub.py +++ b/tests/distributed/omni_coordinator/test_omni_coord_client_for_hub.py @@ -12,6 +12,8 @@ from vllm_omni.distributed.omni_coordinator import ( OmniCoordClientForHub, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _bind_pub() -> tuple[zmq.Context, zmq.Socket, str]: ctx = zmq.Context.instance() diff --git a/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py b/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py index 0ba19c7ff..f095dfd49 100644 --- a/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py +++ b/tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py @@ -17,6 +17,8 @@ from vllm_omni.distributed.omni_coordinator import ( pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]: ctx = zmq.Context.instance() diff --git a/tests/distributed/omni_coordinator/test_omni_coordinator.py b/tests/distributed/omni_coordinator/test_omni_coordinator.py index 0c68e61bb..38a595cc7 100644 --- a/tests/distributed/omni_coordinator/test_omni_coordinator.py +++ b/tests/distributed/omni_coordinator/test_omni_coordinator.py @@ -4,6 +4,7 @@ import json import time +import pytest import zmq from vllm.v1.utils import get_engine_client_zmq_addr @@ -13,6 +14,8 @@ from vllm_omni.distributed.omni_coordinator import ( StageStatus, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _recv_instance_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None: """Receive InstanceList JSON from SUB socket. Returns None on timeout.""" diff --git a/tests/engine/test_output_modality.py b/tests/engine/test_output_modality.py index 5a2a5dfc5..7a9c76502 100644 --- a/tests/engine/test_output_modality.py +++ b/tests/engine/test_output_modality.py @@ -12,6 +12,7 @@ import pytest import torch # ── Load modules without triggering vllm_omni.__init__ ───────────── +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _ENGINE_DIR = Path(__file__).resolve().parents[2] / "vllm_omni" / "engine" diff --git a/tests/entrypoints/openai_api/test_text_splitter.py b/tests/entrypoints/openai_api/test_text_splitter.py index 23d4d191f..a1886662a 100644 --- a/tests/entrypoints/openai_api/test_text_splitter.py +++ b/tests/entrypoints/openai_api/test_text_splitter.py @@ -4,7 +4,7 @@ import pytest from vllm_omni.entrypoints.openai.text_splitter import SentenceSplitter -pytestmark = [pytest.mark.openai, pytest.mark.speech] +pytestmark = [pytest.mark.openai, pytest.mark.speech, pytest.mark.core_model, pytest.mark.cpu] class TestSentenceSplitterEnglish: diff --git a/tests/entrypoints/test_stage_utils.py b/tests/entrypoints/test_stage_utils.py index 2bb2231cc..3afc6f12f 100644 --- a/tests/entrypoints/test_stage_utils.py +++ b/tests/entrypoints/test_stage_utils.py @@ -6,8 +6,6 @@ from pytest_mock import MockerFixture from vllm_omni.entrypoints.stage_utils import set_stage_devices -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - def _make_dummy_torch(call_log): class _Props: @@ -55,6 +53,8 @@ def _make_mock_platform(mocker, device_type: str = "cuda", env_var: str = "CUDA_ return mock_platform +@pytest.mark.core_model +@pytest.mark.cpu @pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): # Preserve an existing logical mapping and ensure devices "0,1" map through it. @@ -75,6 +75,8 @@ def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypat assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7" +@pytest.mark.core_model +@pytest.mark.cpu @pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_handles_not_enough_devices(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): # Preserve an existing logical mapping and ensure devices "0,1" map through it. diff --git a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py index 0f5202c3b..ec24f6949 100644 --- a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py +++ b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py @@ -8,6 +8,8 @@ import pytest import torch import torch.nn as nn +from tests.utils import hardware_test + class TestPreLookaheadLayer: """Tests for PreLookaheadLayer.""" @@ -18,6 +20,8 @@ class TestPreLookaheadLayer: return PreLookaheadLayer(in_channels=512, channels=512, pre_lookahead_len=3) + @pytest.mark.core_model + @pytest.mark.cpu def test_forward_shape(self, layer): """Test that output shape matches input shape.""" batch, seq_len, channels = 2, 10, 512 @@ -27,6 +31,8 @@ class TestPreLookaheadLayer: assert out.shape == x.shape + @pytest.mark.core_model + @pytest.mark.cpu def test_forward_with_context(self, layer): """Test forward with context for streaming.""" batch, seq_len, channels = 1, 10, 512 @@ -38,6 +44,8 @@ class TestPreLookaheadLayer: assert out.shape == x.shape + @pytest.mark.core_model + @pytest.mark.cpu def test_residual_connection(self, layer): """Test that residual connection is applied.""" batch, seq_len, channels = 1, 5, 512 @@ -59,6 +67,8 @@ class TestDiTAttention: return DiTAttention(dim=512, heads=8, dim_head=64, dropout=0.0) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_forward_shape(self, attention): """Test attention output shape.""" batch, seq_len, dim = 2, 16, 512 @@ -68,6 +78,8 @@ class TestDiTAttention: assert out.shape == x.shape + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_forward_with_mask(self, attention): """Test attention with mask.""" batch, seq_len, dim = 2, 16, 512 @@ -81,6 +93,8 @@ class TestDiTAttention: # Masked positions should be zero assert torch.allclose(out[:, -3:], torch.zeros_like(out[:, -3:])) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_qkv_projections(self, attention): """Test that Q/K/V projections exist and have correct dimensions.""" assert hasattr(attention, "to_q") @@ -100,6 +114,8 @@ class TestDiTBlock: return DiTBlock(dim=512, heads=8, dim_head=64, ff_mult=4, dropout=0.0) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_forward_shape(self, block): """Test block output shape.""" batch, seq_len, dim = 2, 16, 512 @@ -110,6 +126,8 @@ class TestDiTBlock: assert out.shape == x.shape + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_adalayernorm_modulation(self, block): """Test that AdaLayerNorm modulates based on timestep.""" batch, seq_len, dim = 1, 8, 512 @@ -144,6 +162,8 @@ class TestDiT: long_skip_connection=True, ) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_forward_shape(self, dit): """Test DiT forward output shape.""" batch, mel_dim, seq_len = 1, 80, 32 @@ -158,6 +178,8 @@ class TestDiT: assert out.shape == (batch, mel_dim, seq_len) + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_timestep_embedding(self, dit): """Test that different timesteps produce different outputs.""" batch, mel_dim, seq_len = 1, 80, 16 @@ -190,6 +212,8 @@ class TestCFM: return DummyEstimator() + @pytest.mark.core_model + @pytest.mark.cpu def test_causal_conditional_cfm_forward(self, dummy_estimator): """Test CausalConditionalCFM forward pass.""" from omegaconf import DictConfig @@ -228,6 +252,8 @@ class TestCFM: class TestSDPAFallback: """Test SDPA fallback for float32 inputs.""" + @pytest.mark.core_model + @hardware_test(res={"cuda": "L4"}, num_cards=1) def test_float32_uses_sdpa(self): """Test that float32 inputs use SDPA fallback.""" from vllm_omni.diffusion.attention.layer import Attention diff --git a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_utils.py b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_utils.py index 828bb2b14..76428ed58 100644 --- a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_utils.py +++ b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_utils.py @@ -5,6 +5,8 @@ import pytest import torch +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class TestMakePadMask: """Tests for make_pad_mask utility.""" diff --git a/tests/test_diffusion_config_propagation.py b/tests/test_diffusion_config_propagation.py index 58eb6097c..7d6d9c43f 100644 --- a/tests/test_diffusion_config_propagation.py +++ b/tests/test_diffusion_config_propagation.py @@ -7,6 +7,7 @@ Regression tests for https://github.com/vllm-project/vllm-omni/issues/1862 from collections.abc import Mapping +import pytest import torch from vllm_omni.config.stage_config import StageConfigFactory @@ -15,6 +16,8 @@ from vllm_omni.diffusion.data import ( OmniDiffusionConfig, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def _roundtrip_diffusion_config(**kwargs) -> OmniDiffusionConfig: """Simulate the real path: create_default_diffusion → OmniDiffusionConfig.