Merge origin/main into dev/rebase-v0.16.0.

Resolve arg_utils merge conflict and align CI base image tag to v0.16.0 for the vLLM rebase track.

Co-authored-by: Cursor <cursoragent@cursor.com>

Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk>
This commit is contained in:
Taichang ZhouandCursor committed 2026-02-12 21:18:03 +08:00
commit e2ced80d59
64 files changed
+5186 -1228

No files matched your search

+578
View File
@@ -0,0 +1,578 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for patch.py
"""
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
class MockResponse:
"""Mock aiohttp response for testing"""
def __init__(self, status, chunks, delay_between_chunks=0):
self.status = status
self.reason = "OK" if status == 200 else "Error"
self._chunks = chunks
self._delay = delay_between_chunks
self.content = self
async def iter_any(self):
for chunk in self._chunks:
if self._delay > 0:
await asyncio.sleep(self._delay)
yield chunk
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
def create_sse_chunk(data_dict):
"""Helper to create SSE formatted chunk"""
return f"data: {json.dumps(data_dict)}\n\n".encode()
# ============================================================================
# output_tokens Tests
# ============================================================================
@pytest.mark.asyncio
async def test_output_tokens_assigned_with_metrics():
"""Test that output.output_tokens is assigned when metrics are present"""
# Arrange
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with metrics
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
"metrics": {"num_tokens_out": 42, "num_tokens_in": 10},
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert output.output_tokens == 42, "output_tokens should be assigned from metrics"
assert output.generated_text == "Hello world"
@pytest.mark.asyncio
async def test_output_tokens_not_assigned_without_metrics():
"""Test that output.output_tokens defaults to 0 when no metrics present"""
# Arrange
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks without metrics
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert output.output_tokens == 0, "output_tokens should default to 0 when no metrics"
assert output.generated_text == "Hello world"
@pytest.mark.asyncio
async def test_output_tokens_assigned_multiple_metrics():
"""Test that output.output_tokens is updated with the latest metrics value"""
# Arrange
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with multiple metrics updates
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
"metrics": {"num_tokens_out": 5},
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
"metrics": {"num_tokens_out": 10},
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": "!"}}],
"modality": "text",
"metrics": {"num_tokens_out": 15},
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert output.output_tokens == 15, "output_tokens should be updated to the latest value"
assert output.generated_text == "Hello world!"
@pytest.mark.asyncio
async def test_output_tokens_with_audio_and_text():
"""Test output_tokens assignment in mixed audio and text response"""
# Arrange
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with both audio and text, with metrics
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Text response"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": ""}}],
"modality": "audio",
}
),
create_sse_chunk(
{
"modality": "text",
"metrics": {"num_tokens_out": 25, "num_tokens_in": 10},
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert output.output_tokens == 25, "output_tokens should be assigned even with audio modality"
@pytest.mark.asyncio
async def test_output_tokens_with_missing_num_tokens_out():
"""Test that output_tokens defaults to 0 when num_tokens_out is missing in metrics"""
# Arrange
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with metrics but without num_tokens_out
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
"metrics": {"num_tokens_in": 10}, # Missing num_tokens_out
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert output.output_tokens == 0, "output_tokens should default to 0 when num_tokens_out is missing"
@pytest.mark.asyncio
async def test_output_tokens_initialization():
"""Test that MixRequestFuncOutput initializes output_tokens correctly"""
# Arrange & Act
output = MixRequestFuncOutput()
# Assert
assert hasattr(output, "output_tokens"), "MixRequestFuncOutput should have output_tokens attribute"
assert output.output_tokens == 0, "output_tokens should be initialized to 0"
# ============================================================================
# text_latency Tests
# ============================================================================
class TestTextLatencyAttribute:
"""Tests for text_latency attribute existence and assignment"""
def test_mix_request_func_output_has_text_latency(self):
"""Test that MixRequestFuncOutput has text_latency attribute"""
output = MixRequestFuncOutput()
assert hasattr(output, "text_latency"), "MixRequestFuncOutput should have text_latency attribute"
def test_text_latency_initial_value(self):
"""Test that text_latency initializes to a default value"""
output = MixRequestFuncOutput()
# Check if attribute exists and has a value (should be 0.0 or similar default)
text_latency = getattr(output, "text_latency", None)
assert text_latency is not None or hasattr(output, "text_latency"), "text_latency attribute should exist"
@pytest.mark.asyncio
async def test_text_latency_assigned_with_text_response(self):
"""Test that text_latency is assigned when text response is received"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with text modality
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "Output should have text_latency attribute"
assert output.text_latency > 0, "text_latency should be greater than 0 for text response"
@pytest.mark.asyncio
async def test_text_latency_updated_with_multiple_text_chunks(self):
"""Test that text_latency is updated as more text chunks arrive"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with small delays to simulate streaming
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "First"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " second"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " third"}}],
"modality": "text",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "Output should have text_latency attribute"
assert output.text_latency > 0, "text_latency should accumulate"
assert output.generated_text == "First second third"
@pytest.mark.asyncio
async def test_text_latency_with_only_audio_response(self):
"""Test text_latency behavior when only audio is received"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response chunks with only audio modality (no text)
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": ""}}],
"modality": "audio",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "Output should have text_latency attribute even with audio-only"
# text_latency should either be 0 or the initial value since no text was processed
assert output.text_latency >= 0, "text_latency should be non-negative"
@pytest.mark.asyncio
async def test_text_latency_not_affected_by_metrics(self):
"""Test that text_latency is independent of metrics data"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response with text and metrics
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Response text"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": ""}}],
"modality": "text",
"metrics": {"num_tokens_out": 100, "num_tokens_in": 20},
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "text_latency should exist"
assert output.text_latency > 0, "text_latency should be set for text response"
# Verify metrics are also present
assert output.output_tokens == 100, "metrics should not affect text_latency"
@pytest.mark.asyncio
async def test_text_latency_mixed_modalities(self):
"""Test text_latency with mixed text and audio modalities"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
# Create response with both text and audio
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Text"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": ""}}],
"modality": "audio",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " more text"}}],
"modality": "text",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "text_latency should exist with mixed modalities"
assert output.text_latency > 0, "text_latency should be set when text is present"
assert output.generated_text == "Text more text"
@pytest.mark.asyncio
async def test_text_latency_value_consistency(self):
"""Test that text_latency matches latency minus ttft relationship"""
request_input = RequestFuncInput(
model="test-model",
model_name="test-model",
prompt="test prompt",
api_url="http://test.com/v1/chat/completions",
prompt_len=10,
output_len=20,
)
chunks = [
create_sse_chunk(
{
"choices": [{"delta": {"content": "Hello"}}],
"modality": "text",
}
),
create_sse_chunk(
{
"choices": [{"delta": {"content": " world"}}],
"modality": "text",
}
),
b"data: [DONE]\n\n",
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
# Assert
assert output.success is True
assert hasattr(output, "text_latency"), "text_latency should exist"
assert hasattr(output, "ttft"), "ttft should exist"
assert hasattr(output, "latency"), "latency should exist"
# text_latency should be between ttft and total latency
assert output.ttft <= output.text_latency <= output.latency, (
"text_latency should be between ttft and total latency"
)
if __name__ == "__main__":
pytest.main([__file__, "-v", "-s"])
+54 -21
View File
@@ -36,7 +36,7 @@ def mock_od_config():
@pytest.fixture
def mock_gpu_worker(mock_od_config):
"""Create a DiffusionWorker with mocked initialization."""
with patch.object(DiffusionWorker, "init_device"):
with patch.object(DiffusionWorker, "init_device"), patch.object(DiffusionWorker, "load_model"):
worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config)
# Mock the model_runner with pipeline
worker.model_runner = Mock()
@@ -81,18 +81,18 @@ class TestDiffusionWorkerLoadWeights:
class TestDiffusionWorkerSleep:
"""Test DiffusionWorker.sleep method."""
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_level_1(self, mock_allocator_class, mock_platform, mock_gpu_worker):
def test_sleep_level_1(self, mock_allocator_class, mock_platform, mock_get_process_memory, mock_gpu_worker):
"""Test sleep mode level 1 (offload weights only)."""
# Setup memory info mocks
# Before sleep: 1GB free
# After sleep: 3GB free (freed 2GB)
mock_platform.get_free_memory.side_effect = [
1 * 1024**3, # Before sleep
3 * 1024**3, # After sleep
# Setup process-scoped memory mocks
# Before sleep: 3GB used
# After sleep: 1GB used (freed 2GB)
mock_get_process_memory.side_effect = [
3 * 1024**3,
1 * 1024**3,
]
mock_platform.get_device_total_memory.return_value = 8 * 1024**3
# Setup allocator mock
mock_allocator = Mock()
@@ -108,16 +108,16 @@ class TestDiffusionWorkerSleep:
# Verify buffers were NOT saved (level 1 doesn't save buffers)
assert len(mock_gpu_worker._sleep_saved_buffers) == 0
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_level_2(self, mock_allocator_class, mock_platform, mock_gpu_worker):
def test_sleep_level_2(self, mock_allocator_class, mock_platform, mock_get_process_memory, mock_gpu_worker):
"""Test sleep mode level 2 (offload all, save buffers)."""
# Setup memory info mocks
mock_platform.get_free_memory.side_effect = [
1 * 1024**3, # Before sleep
5 * 1024**3, # After sleep (freed 4GB)
# Setup process-scoped memory mocks
mock_get_process_memory.side_effect = [
5 * 1024**3, # Before sleep
1 * 1024**3, # After sleep (freed 4GB)
]
mock_platform.get_device_total_memory.return_value = 8 * 1024**3
# Setup allocator mock
mock_allocator = Mock()
@@ -146,16 +146,22 @@ class TestDiffusionWorkerSleep:
assert "buffer1" in mock_gpu_worker._sleep_saved_buffers
assert "buffer2" in mock_gpu_worker._sleep_saved_buffers
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_memory_freed_validation(self, mock_allocator_class, mock_platform, mock_gpu_worker):
def test_sleep_memory_freed_validation(
self,
mock_allocator_class,
mock_platform,
mock_get_process_memory,
mock_gpu_worker,
):
"""Test that sleep validates memory was actually freed."""
# Simulate memory increase (should trigger assertion error)
mock_platform.get_free_memory.side_effect = [
3 * 1024**3, # Before sleep: 3GB free
1 * 1024**3, # After sleep: 1GB free (negative freed!)
# Simulate process memory increase (should trigger assertion error)
mock_get_process_memory.side_effect = [
1 * 1024**3, # Before sleep: 1GB used
3 * 1024**3, # After sleep: 3GB used (negative freed)
]
mock_platform.get_device_total_memory.return_value = 8 * 1024**3
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
@@ -165,6 +171,33 @@ class TestDiffusionWorkerSleep:
with pytest.raises(AssertionError, match="Memory usage increased after sleeping"):
mock_gpu_worker.sleep(level=1)
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_falls_back_to_device_memory_when_nvml_unavailable(
self,
mock_allocator_class,
mock_platform,
mock_get_process_memory,
mock_gpu_worker,
):
"""Test sleep uses device-scoped fallback when NVML is unavailable."""
mock_get_process_memory.side_effect = [None, None]
mock_platform.get_free_memory.side_effect = [
1 * 1024**3, # Before sleep
3 * 1024**3, # After sleep
]
mock_platform.get_device_total_memory.return_value = 8 * 1024**3
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.sleep = Mock()
result = mock_gpu_worker.sleep(level=1)
mock_allocator.sleep.assert_called_once_with(offload_tags=("weights",))
assert result is True
class TestDiffusionWorkerWakeUp:
"""Test DiffusionWorker.wake_up method."""
+540
View File
@@ -0,0 +1,540 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for WorkerWrapperBase class.
This module tests the WorkerWrapperBase implementation:
- Initialization with and without worker extensions
- Custom pipeline initialization
- Method delegation via execute_method
- Attribute delegation via __getattr__
- Dynamic worker class extension
"""
from typing import Any
from unittest.mock import Mock, patch
import pytest
from vllm_omni.diffusion.worker.diffusion_worker import (
CustomPipelineWorkerExtension,
DiffusionWorker,
WorkerWrapperBase,
)
# -------------------------------------------------------------------------
# Fixtures
# -------------------------------------------------------------------------
@pytest.fixture
def mock_od_config():
"""Create a mock OmniDiffusionConfig for use in tests."""
config = Mock()
config.num_gpus = 1
config.master_port = 12345
config.enable_sleep_mode = False
config.cache_backend = None
config.cache_config = None
config.model = "test-model"
config.diffusion_load_format = None
config.dtype = "float32"
config.max_cpu_loras = 0
config.lora_path = None
config.lora_scale = 1.0
return config
class TestExtension:
"""Simple test extension adding one custom method."""
def custom_method(self):
return "extension_method"
class MockCustomPipeline:
"""Mock custom pipeline for testing."""
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
def __call__(self, *args, **kwargs):
return "pipeline_output"
# -------------------------------------------------------------------------
# Tests: Initialization
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseInitialization:
"""Test WorkerWrapperBase initialization behavior."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_basic_initialization(self, mock_worker_init, mock_od_config):
"""Test basic initialization without extensions."""
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
)
assert wrapper.gpu_id == 0
assert wrapper.od_config == mock_od_config
assert wrapper.base_worker_class == DiffusionWorker
assert wrapper.worker_extension_cls is None
assert wrapper.custom_pipeline_args is None
assert wrapper.worker is not None
mock_worker_init.assert_called_once_with(
local_rank=0,
rank=0,
od_config=mock_od_config,
)
# -------------------------------------------------------------------------
# Tests: Worker Extension Functionality
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseExtension:
"""Test WorkerWrapperBase worker extension functionality."""
def test_prepare_worker_class_without_extension(self, mock_od_config):
"""Test _prepare_worker_class without a worker extension."""
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
)
worker_class = wrapper._prepare_worker_class()
assert worker_class == DiffusionWorker
def test_prepare_worker_class_with_extension_class(self, mock_od_config):
"""Test _prepare_worker_class with an explicit extension class."""
class TestExtension:
def custom_method(self):
return "extension_method"
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
assert hasattr(wrapper.worker.__class__, "custom_method")
assert TestExtension in wrapper.worker.__class__.__bases__
@patch("vllm.utils.import_utils.resolve_obj_by_qualname")
def test_prepare_worker_class_with_extension_string(self, mock_resolve, mock_od_config):
"""Test _prepare_worker_class with worker extension as string."""
mock_resolve.return_value = TestExtension
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls="tests.diffusion.test_worker_wrapper_base.TestExtension",
)
assert hasattr(wrapper.worker.__class__, "custom_method")
# -------------------------------------------------------------------------
# Tests: Method Delegation
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseDelegation:
"""Test WorkerWrapperBase delegation to wrapped worker."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_generate_delegation(self, mock_worker_init, mock_od_config):
"""Test that generate() delegates to worker.generate()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
mock_output = Mock()
wrapper.worker.generate = Mock(return_value=mock_output)
mock_requests = [Mock()]
result = wrapper.generate(mock_requests)
wrapper.worker.generate.assert_called_once_with(mock_requests)
assert result == mock_output
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_model_delegation(self, mock_worker_init, mock_od_config):
"""Test that execute_model() delegates to worker.execute_model()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
mock_output = Mock()
wrapper.worker.execute_model = Mock(return_value=mock_output)
mock_reqs = [Mock()]
result = wrapper.execute_model(mock_reqs, mock_od_config)
wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config)
assert result == mock_output
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_load_weights_delegation(self, mock_worker_init, mock_od_config):
"""Test that load_weights() delegates to worker.load_weights()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
expected_result = {"weight1", "weight2"}
wrapper.worker.load_weights = Mock(return_value=expected_result)
mock_weights = [("weight1", Mock()), ("weight2", Mock())]
result = wrapper.load_weights(mock_weights)
wrapper.worker.load_weights.assert_called_once_with(mock_weights)
assert result == expected_result
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_sleep_delegation(self, mock_worker_init, mock_od_config):
"""Test that sleep() delegates to worker.sleep()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.sleep = Mock(return_value=True)
result = wrapper.sleep(level=1)
wrapper.worker.sleep.assert_called_once_with(1)
assert result is True
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_wake_up_delegation(self, mock_worker_init, mock_od_config):
"""Test that wake_up() delegates to worker.wake_up()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.wake_up = Mock(return_value=True)
result = wrapper.wake_up(tags=["weights"])
wrapper.worker.wake_up.assert_called_once_with(["weights"])
assert result is True
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_shutdown_delegation(self, mock_worker_init, mock_od_config):
"""Test that shutdown() delegates to worker.shutdown()."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.shutdown = Mock(return_value=None)
result = wrapper.shutdown()
wrapper.worker.shutdown.assert_called_once()
assert result is None
# -------------------------------------------------------------------------
# Tests: execute_method
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseExecuteMethod:
"""Test WorkerWrapperBase.execute_method functionality."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_success(self, mock_worker_init, mock_od_config):
"""Test execute_method successfully calls worker method."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.test_method = Mock(return_value="method_result")
result = wrapper.execute_method("test_method", "arg1", kwarg1="value1")
wrapper.worker.test_method.assert_called_once_with("arg1", kwarg1="value1")
assert result == "method_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_with_no_args(self, mock_worker_init, mock_od_config):
"""Test execute_method with no arguments."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.no_args_method = Mock(return_value="no_args_result")
result = wrapper.execute_method("no_args_method")
wrapper.worker.no_args_method.assert_called_once_with()
assert result == "no_args_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_error(self, mock_worker_init, mock_od_config):
"""Test execute_method raises exception on error."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.error_method = Mock(side_effect=RuntimeError("Test error"))
with pytest.raises(RuntimeError, match="Test error"):
wrapper.execute_method("error_method")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_invalid_type(self, mock_worker_init, mock_od_config):
"""Test execute_method with invalid method type."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
with pytest.raises(AssertionError, match="Method must be str"):
wrapper.execute_method(b"bytes_method")
# -------------------------------------------------------------------------
# Tests: __getattr__ delegation
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseGetAttr:
"""Test WorkerWrapperBase.__getattr__ delegation."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_delegation(self, mock_worker_init, mock_od_config):
"""Test __getattr__ delegates to worker attributes."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.custom_attribute = "test_value"
assert wrapper.custom_attribute == "test_value"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_method_access(self, mock_worker_init, mock_od_config):
"""Test __getattr__ delegates to worker methods."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.custom_method = Mock(return_value="method_result")
result = wrapper.custom_method()
wrapper.worker.custom_method.assert_called_once()
assert result == "method_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_missing_attribute(self, mock_worker_init, mock_od_config):
"""Test __getattr__ raises AttributeError for missing attributes."""
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
with pytest.raises(AttributeError):
_ = wrapper.nonexistent_attribute
# -------------------------------------------------------------------------
# Tests: Edge Cases
# -------------------------------------------------------------------------
class TestWorkerWrapperBaseEdgeCases:
"""Test WorkerWrapperBase edge cases and special scenarios."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_extension_conflict_warning(self, mock_worker_init, mock_od_config, caplog):
"""Test a warning is logged when an extension conflicts with worker."""
class ConflictExtension:
def load_model(self):
return "extension_load_model"
with patch.object(DiffusionWorker, "load_model"):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=ConflictExtension,
)
assert wrapper.worker is not None
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_multiple_extensions_same_class(self, mock_worker_init, mock_od_config):
"""Test that applying same extension twice doesn't duplicate it."""
class TestExtension:
def custom_method(self):
return "extension"
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper1 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
wrapper2 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
assert hasattr(wrapper1.worker, "custom_method")
assert hasattr(wrapper2.worker, "custom_method")
# -------------------------------------------------------------------------
# Tests: CustomPipelineWorkerExtension
# -------------------------------------------------------------------------
class TestCustomPipelineWorkerExtension:
"""Test CustomPipelineWorkerExtension functionality."""
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_basic(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
"""Test basic re_init_pipeline functionality."""
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomPipelineWorkerExtension,
)
# Setup mock model_runner and pipeline
mock_model_runner = Mock()
mock_pipeline = Mock()
mock_model_runner.pipeline = mock_pipeline
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
# Call re_init_pipeline
wrapper.worker.re_init_pipeline(custom_args)
# Verify load_model was called with correct arguments
wrapper.worker.load_model.assert_called_once_with(
load_format="custom_pipeline",
custom_pipeline_name="tests.diffusion.test_worker_wrapper_base.MockCustomPipeline",
)
wrapper.worker.init_lora_manager.assert_called_once()
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_cleanup(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
"""Test that re_init_pipeline properly cleans up old pipeline."""
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomPipelineWorkerExtension,
)
# Setup mock model_runner with pipeline
mock_model_runner = Mock()
mock_pipeline = Mock()
mock_model_runner.pipeline = mock_pipeline
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
# Call re_init_pipeline
wrapper.worker.re_init_pipeline(custom_args)
# Verify cleanup was performed
mock_gc_collect.assert_called_once()
mock_empty_cache.assert_called_once()
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_none_pipeline(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
"""Test re_init_pipeline when pipeline is None."""
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomPipelineWorkerExtension,
)
# Setup mock model_runner with None pipeline
mock_model_runner = Mock()
mock_model_runner.pipeline = None
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
# Should not raise an error
wrapper.worker.re_init_pipeline(custom_args)
# Verify load_model was still called
wrapper.worker.load_model.assert_called_once()
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_custom_pipeline_args_initialization(self, mock_worker_init, mock_od_config):
"""Test initialization with custom_pipeline_args calls re_init_pipeline."""
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
with patch.object(DiffusionWorker, "__init__", return_value=None):
with patch.object(WorkerWrapperBase, "_prepare_worker_class") as mock_prepare:
# Create a mock worker class with re_init_pipeline
mock_worker_class = Mock()
mock_worker_instance = Mock()
mock_worker_instance.re_init_pipeline = Mock()
mock_worker_class.return_value = mock_worker_instance
mock_prepare.return_value = mock_worker_class
_ = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
custom_pipeline_args=custom_args,
)
# Verify re_init_pipeline was called with custom_pipeline_args
mock_worker_instance.re_init_pipeline.assert_called_once_with(custom_args)
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_custom_pipeline_with_explicit_extension(self, mock_worker_init, mock_od_config):
"""Test that explicit worker_extension_cls is preserved when custom_pipeline_args is provided."""
class CustomExtension:
def re_init_pipeline(self, custom_pipeline_args: dict[str, Any]):
return "custom_re_init_pipeline"
def custom_extension_method(self):
return "custom_extension_method"
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomExtension,
custom_pipeline_args=custom_args,
)
# Should still have the explicitly provided extension
assert CustomExtension in wrapper.worker.__class__.__bases__
assert hasattr(wrapper.worker, "custom_extension_method")
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_multiple_calls(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
"""Test calling re_init_pipeline multiple times."""
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomPipelineWorkerExtension,
)
# Setup mock model_runner
mock_model_runner = Mock()
mock_pipeline1 = Mock()
mock_pipeline2 = Mock()
mock_model_runner.pipeline = mock_pipeline1
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
# First call
custom_args1 = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
wrapper.worker.re_init_pipeline(custom_args1)
# Update pipeline for second call
mock_model_runner.pipeline = mock_pipeline2
# Second call
custom_args2 = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
wrapper.worker.re_init_pipeline(custom_args2)
# Verify load_model was called twice with different pipelines
assert wrapper.worker.load_model.call_count == 2
assert wrapper.worker.init_lora_manager.call_count == 2
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from collections import deque
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.v1.request import RequestStatus
from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
from vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter import (
OmniChunkTransferAdapter,
)
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class DummyWaitingQueue(list):
def prepend_requests(self, requests):
self[:0] = list(requests)
def add_request(self, request):
self.append(request)
def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None):
return SimpleNamespace(
request_id=req_id,
external_req_id=external_req_id or req_id,
status=status,
prompt_token_ids=[],
num_computed_tokens=0,
additional_information=None,
)
@pytest.fixture
def build_adapter(monkeypatch):
def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2):
connector = MagicMock()
connector.stage_id = stage_id
connector.get.return_value = None
connector.put.return_value = (True, 1, {})
def _fake_base_init(self, config):
self.config = config
self._pending_load_reqs = {}
self._finished_load_reqs = set()
self._pending_save_reqs = {}
self._finished_save_reqs = set()
self.stop_event = threading.Event()
self.lock = threading.Lock()
monkeypatch.setattr(OmniTransferAdapterBase, "__init__", _fake_base_init)
monkeypatch.setattr(
OmniChunkTransferAdapter,
"create_connector",
classmethod(lambda cls, _model_config: connector),
)
model_config = SimpleNamespace(worker_type=model_mode)
scheduler_config = SimpleNamespace(max_num_seqs=max_num_seqs)
adapter = OmniChunkTransferAdapter(
SimpleNamespace(model_config=model_config, scheduler_config=scheduler_config)
)
return adapter, connector
return _build
@pytest.mark.parametrize(
("raw_cfg", "expected_name", "expected_extra"),
[
(None, "SharedMemoryConnector", {}),
(SimpleNamespace(name="YuanrongConnector", extra={"k": "v"}), "YuanrongConnector", {"k": "v"}),
],
)
def test_create_connector_config_parsing(monkeypatch, raw_cfg, expected_name, expected_extra):
captured = {}
def _fake_create(spec):
captured["spec"] = spec
return "ok"
monkeypatch.setattr(
"vllm_omni.distributed.omni_connectors.transfer_adapter.chunk_transfer_adapter"
".OmniConnectorFactory.create_connector",
_fake_create,
)
model_config = SimpleNamespace(stage_connector_config=raw_cfg) if raw_cfg is not None else SimpleNamespace()
connector = OmniChunkTransferAdapter.create_connector(model_config)
assert connector == "ok"
assert isinstance(captured["spec"], ConnectorSpec)
assert captured["spec"].name == expected_name
assert captured["spec"].extra == expected_extra
def test_load_poll(build_adapter):
adapter, connector = build_adapter(stage_id=2, model_mode="ar")
request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1")
adapter.load_async(request)
payload = {"code_predictor_codes": [[1]], "hidden_states": torch.tensor([[2.0]]), "finished": True}
connector.get.return_value = (payload, 16)
adapter._poll_single_request("req-1")
connector.get.assert_called_once_with("1", "2", "external-1_1_0")
assert request.additional_information == payload
assert adapter.get_req_chunk["req-1"] == 1
assert "req-1" in adapter._finished_load_reqs
assert "req-1" in adapter.finished_requests
assert "req-1" not in adapter._pending_load_reqs
def test_save_async(build_adapter):
adapter, _ = build_adapter(stage_id=1)
request = SimpleNamespace(external_req_id="external-1")
adapter.custom_process_next_stage_input_func = lambda **kwargs: {"x": [1], "finished": False}
adapter.save_async(pooling_output=None, request=request)
adapter.custom_process_next_stage_input_func = lambda **kwargs: {}
adapter.save_async(pooling_output=None, request=request)
assert adapter.put_req_chunk["external-1"] == 1
queued = adapter._pending_save_reqs["external-1"]
assert len(queued) == 1
assert queued[0]["put_key"] == "external-1_1_0"
def test_update_request_payload(build_adapter):
adapter, _ = build_adapter()
adapter._update_request_payload("ext", {"h": torch.tensor([[1.0]]), "codes": [1], "finished": False})
merged = adapter._update_request_payload("ext", {"h": torch.tensor([[2.0]]), "codes": [2], "finished": True})
assert torch.equal(merged["h"], torch.tensor([[1.0], [2.0]]))
assert merged["codes"] == [1, 2]
assert merged["finished"] is True
def test_process_and_restore_queues(build_adapter):
adapter, _ = build_adapter(stage_id=1, max_num_seqs=8)
waiting_req = _req("w1", RequestStatus.WAITING)
running_req = _req("r1", RequestStatus.RUNNING)
waiting_queue = DummyWaitingQueue([waiting_req])
running_queue = [running_req]
adapter.process_pending_chunks(waiting_queue, running_queue)
assert waiting_req.status == RequestStatus.WAITING_FOR_CHUNK
assert running_req.status == RequestStatus.WAITING_FOR_CHUNK
assert waiting_queue == []
assert running_queue == []
adapter.restore_queues(waiting_queue, running_queue)
assert waiting_queue == [waiting_req]
assert running_queue == [running_req]
assert adapter.waiting_for_chunk_waiting_requests == deque()
assert adapter.waiting_for_chunk_running_requests == deque()
def test_postprocess_scheduler_output(build_adapter):
adapter, _ = build_adapter()
adapter.requests_with_ready_chunks = {"new-ready", "cached-ready", "leftover"}
scheduler_output = SimpleNamespace(
scheduled_new_reqs=[SimpleNamespace(req_id="new-ready")],
scheduled_cached_reqs=SimpleNamespace(req_ids=["cached-ready", "missing"]),
)
requests = {"cached-ready": SimpleNamespace(additional_information={"k": "v"})}
adapter.postprocess_scheduler_output(scheduler_output, requests)
cached_info = scheduler_output.scheduled_cached_reqs.additional_information
assert cached_info["cached-ready"] == {"k": "v"}
assert cached_info["missing"] is None
assert adapter.requests_with_ready_chunks == {"leftover"}
@@ -17,13 +17,15 @@ if str(REPO_ROOT) not in sys.path:
from vllm_omni import Omni
# Models to test and expected saved memory in MB, correspondingly
MODELS_SAVED_MEMORY_MB = {"riverclouds/qwen_image_random": 4500}
MODELS_SAVED_MEMORY_MB = {
"riverclouds/qwen_image_random": 4500,
# "Wan-AI/Wan2.2-T2V-A14B-Diffusers": 45000,
}
def run_inference(
model_name: str,
layerwise_offload: bool = False,
num_gpu_layers: int = 1,
num_inference_steps: int = 3,
) -> float:
# For now, only support on GPU, so apply torch.cuda operations here
@@ -36,7 +38,6 @@ def run_inference(
m = Omni(
model=model_name,
enable_layerwise_offload=layerwise_offload,
layerwise_num_gpu_layers=num_gpu_layers,
boundary_ratio=0.875,
flow_shift=5.0,
)
@@ -83,16 +84,12 @@ def test_layerwise_offload_diffusion_model(model_name: str):
cleanup_dist_env_and_memory()
# Run with layerwise offloading (1 layer on device)
layerwise_offload_peak_memory = run_inference(model_name, layerwise_offload=True, num_gpu_layers=1)
layerwise_offload_peak_memory = run_inference(model_name, layerwise_offload=True)
cleanup_dist_env_and_memory()
# Run with 2 layers on device
layerwise_offload_two_layers_peak = run_inference(model_name, layerwise_offload=True, num_gpu_layers=2)
except Exception:
pytest.fail("Inference failed")
print(f"Layerwise offload peak memory (1 GPU layer): {layerwise_offload_peak_memory} MB")
print(f"Layerwise offload peak memory (2 GPU layers): {layerwise_offload_two_layers_peak} MB")
print(f"No offload peak memory: {no_offload_peak_memory} MB")
# Verify that layerwise offloading significantly reduces memory usage
@@ -101,10 +98,3 @@ def test_layerwise_offload_diffusion_model(model_name: str):
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB"
)
# Verify that 2 GPU layers uses more memory than 1 GPU layer
# But not excessively more (should be a reasonable increase)
assert layerwise_offload_peak_memory < layerwise_offload_two_layers_peak, (
f"1 GPU layer peak {layerwise_offload_peak_memory} MB should be < "
f"2 GPU layers peak {layerwise_offload_two_layers_peak} MB"
)
@@ -34,10 +34,17 @@ elif current_omni_platform.is_rocm():
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 2})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_model(model_name: str):
def test_diffusion_model(model_name: str, run_level):
if run_level == "core_model" and model_name != "riverclouds/qwen_image_random":
pytest.skip()
if run_level == "advanced_model" and model_name == "riverclouds/qwen_image_random":
pytest.skip()
m = None
try:
m = Omni(model=model_name)
@@ -154,7 +154,7 @@ def _run_zimage_generate(
cleanup_dist_env_and_memory()
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@@ -145,7 +145,7 @@ def _basic_payload() -> dict:
}
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
def test_images_generations_per_request_lora_switching(omni_server: OmniServer, tmp_path: Path) -> None:
@@ -10,7 +10,7 @@ stage_args:
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"