Merge origin/main into dev/vllm-align

Resolved conflicts:
- vllm_omni/inputs/preprocess.py: took origin/main's elif mm_processor_kwargs branch, fixed undefined mm_uuids reference
- vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py: took origin/main's NPU support

Made-with: Cursor

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-03-02 02:10:32 +00:00
commit 26744bcb35
240 files changed
+25869 -3091

No files matched your search

+34 -34
View File
@@ -7,9 +7,9 @@ Unit tests for patch.py
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
@@ -51,7 +51,7 @@ def create_sse_chunk(data_dict):
@pytest.mark.asyncio
async def test_output_tokens_assigned_with_metrics():
async def test_output_tokens_assigned_with_metrics(mocker: MockerFixture):
"""Test that output.output_tokens is assigned when metrics are present"""
# Arrange
request_input = RequestFuncInput(
@@ -82,8 +82,8 @@ async def test_output_tokens_assigned_with_metrics():
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -95,7 +95,7 @@ async def test_output_tokens_assigned_with_metrics():
@pytest.mark.asyncio
async def test_output_tokens_not_assigned_without_metrics():
async def test_output_tokens_not_assigned_without_metrics(mocker: MockerFixture):
"""Test that output.output_tokens defaults to 0 when no metrics present"""
# Arrange
request_input = RequestFuncInput(
@@ -125,8 +125,8 @@ async def test_output_tokens_not_assigned_without_metrics():
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -138,7 +138,7 @@ async def test_output_tokens_not_assigned_without_metrics():
@pytest.mark.asyncio
async def test_output_tokens_assigned_multiple_metrics():
async def test_output_tokens_assigned_multiple_metrics(mocker: MockerFixture):
"""Test that output.output_tokens is updated with the latest metrics value"""
# Arrange
request_input = RequestFuncInput(
@@ -177,8 +177,8 @@ async def test_output_tokens_assigned_multiple_metrics():
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -190,7 +190,7 @@ async def test_output_tokens_assigned_multiple_metrics():
@pytest.mark.asyncio
async def test_output_tokens_with_audio_and_text():
async def test_output_tokens_with_audio_and_text(mocker: MockerFixture):
"""Test output_tokens assignment in mixed audio and text response"""
# Arrange
request_input = RequestFuncInput(
@@ -226,8 +226,8 @@ async def test_output_tokens_with_audio_and_text():
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -238,7 +238,7 @@ async def test_output_tokens_with_audio_and_text():
@pytest.mark.asyncio
async def test_output_tokens_with_missing_num_tokens_out():
async def test_output_tokens_with_missing_num_tokens_out(mocker: MockerFixture):
"""Test that output_tokens defaults to 0 when num_tokens_out is missing in metrics"""
# Arrange
request_input = RequestFuncInput(
@@ -269,8 +269,8 @@ async def test_output_tokens_with_missing_num_tokens_out():
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -312,7 +312,7 @@ class TestTextLatencyAttribute:
assert text_latency is not None or hasattr(output, "text_latency"), "text_latency attribute should exist"
@pytest.mark.asyncio
async def test_text_latency_assigned_with_text_response(self):
async def test_text_latency_assigned_with_text_response(self, mocker: MockerFixture):
"""Test that text_latency is assigned when text response is received"""
request_input = RequestFuncInput(
model="test-model",
@@ -341,8 +341,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -353,7 +353,7 @@ class TestTextLatencyAttribute:
assert output.text_latency > 0, "text_latency should be greater than 0 for text response"
@pytest.mark.asyncio
async def test_text_latency_updated_with_multiple_text_chunks(self):
async def test_text_latency_updated_with_multiple_text_chunks(self, mocker: MockerFixture):
"""Test that text_latency is updated as more text chunks arrive"""
request_input = RequestFuncInput(
model="test-model",
@@ -388,8 +388,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -401,7 +401,7 @@ class TestTextLatencyAttribute:
assert output.generated_text == "First second third"
@pytest.mark.asyncio
async def test_text_latency_with_only_audio_response(self):
async def test_text_latency_with_only_audio_response(self, mocker: MockerFixture):
"""Test text_latency behavior when only audio is received"""
request_input = RequestFuncInput(
model="test-model",
@@ -424,8 +424,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -437,7 +437,7 @@ class TestTextLatencyAttribute:
assert output.text_latency >= 0, "text_latency should be non-negative"
@pytest.mark.asyncio
async def test_text_latency_not_affected_by_metrics(self):
async def test_text_latency_not_affected_by_metrics(self, mocker: MockerFixture):
"""Test that text_latency is independent of metrics data"""
request_input = RequestFuncInput(
model="test-model",
@@ -467,8 +467,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -481,7 +481,7 @@ class TestTextLatencyAttribute:
assert output.output_tokens == 100, "metrics should not affect text_latency"
@pytest.mark.asyncio
async def test_text_latency_mixed_modalities(self):
async def test_text_latency_mixed_modalities(self, mocker: MockerFixture):
"""Test text_latency with mixed text and audio modalities"""
request_input = RequestFuncInput(
model="test-model",
@@ -516,8 +516,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
@@ -529,7 +529,7 @@ class TestTextLatencyAttribute:
assert output.generated_text == "Text more text"
@pytest.mark.asyncio
async def test_text_latency_value_consistency(self):
async def test_text_latency_value_consistency(self, mocker: MockerFixture):
"""Test that text_latency matches latency minus ttft relationship"""
request_input = RequestFuncInput(
model="test-model",
@@ -557,8 +557,8 @@ class TestTextLatencyAttribute:
]
mock_response = MockResponse(200, chunks)
mock_session = AsyncMock()
mock_session.post = MagicMock(return_value=mock_response)
mock_session = mocker.AsyncMock()
mock_session.post = mocker.MagicMock(return_value=mock_response)
# Act
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
+3 -21
View File
@@ -3,36 +3,18 @@ from pathlib import Path
import pytest
from tests.conftest import OmniServer
from tests.utils import hardware_test
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen3_omni_ci.yaml")]
models = ["Qwen/Qwen2.5-Omni-7B"]
stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_ci.yaml")]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
@pytest.fixture(scope="module")
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
model, stage_config_path = request.param
print(f"Starting OmniServer with model: {model}")
print("This may take 10-20+ minutes for initialization...")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
print("OmniServer started successfully")
yield server
print("OmniServer stopped")
@pytest.mark.core_model
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@hardware_test(res={"cuda": "L4"}, num_cards=3)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_bench_serve_chat(omni_server):
command = [
+82
View File
@@ -0,0 +1,82 @@
"""
Conftest for ComfyUI-vLLM-Omni tests.
This module sets up the test environment by:
1. Adding the ComfyUI plugin to Python path
2. Mocking comfy_api.input module (AudioInput, VideoInput) since comfyui is not installed
3. Mocking comfy_extras.nodes_audio module
"""
import os
import sys
from typing import BinaryIO, TypedDict
from unittest.mock import MagicMock
def pytest_configure(config):
"""
Called after command line options have been parsed and before test collection.
This is the right place to set up sys.path and mock modules.
"""
_setup_comfyui_test_environment()
def _setup_comfyui_test_environment():
"""Set up the test environment for ComfyUI plugin testing."""
# === Add ComfyUI plugin path to allow importing comfyui_vllm_omni ===
_COMFYUI_PLUGIN_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "apps", "ComfyUI-vLLM-Omni")
)
if not os.path.isdir(_COMFYUI_PLUGIN_PATH):
raise FileNotFoundError(
f"ComfyUI plugin not found at {_COMFYUI_PLUGIN_PATH}. "
"If it is moved elsewhere, please update the path in this conftest.py."
)
if _COMFYUI_PLUGIN_PATH not in sys.path:
sys.path.insert(0, _COMFYUI_PLUGIN_PATH)
# Import torch after changing import paths. (To be used later)
import torch
# === Mock ComfyUI internal modules (comfy_api & comfy_extras) and "import" them to sys.module ===
class AudioInput(TypedDict):
"""Mock AudioInput TypedDict from comfy_api.input"""
waveform: torch.Tensor # Shape: (B, C, T)
sample_rate: int
class VideoInput:
"""Mock VideoInput class from comfy_api.input"""
def __init__(self, data: bytes = b"mock_video_data"):
self._data = data
def save_to(self, file: str | BinaryIO):
"""Save video data to file or file-like object."""
if isinstance(file, str):
print("Called VideoInput.save_to with file path. Saving to a path is no-op in tests.")
else:
file.write(self._data)
mock_comfy_api = MagicMock()
mock_comfy_api_input = MagicMock()
mock_comfy_api_input.AudioInput = AudioInput
mock_comfy_api_input.VideoInput = VideoInput
mock_comfy_api.input = mock_comfy_api_input
def mock_load(_: str | BinaryIO):
"""Mock nodes_audio.load that returns a waveform tensor (channels, samples) and sample rate."""
waveform = torch.zeros((1, 24000), dtype=torch.float32)
sample_rate = 24000
return waveform, sample_rate
mock_comfy_extras = MagicMock()
mock_nodes_audio = MagicMock()
mock_nodes_audio.load = mock_load
mock_comfy_extras.nodes_audio = mock_nodes_audio
# Install mock modules BEFORE importing any comfyui_vllm_omni code
sys.modules["comfy_api"] = mock_comfy_api
sys.modules["comfy_api.input"] = mock_comfy_api_input
sys.modules["comfy_extras"] = mock_comfy_extras
sys.modules["comfy_extras.nodes_audio"] = mock_nodes_audio
+567
View File
@@ -0,0 +1,567 @@
"""
Integration tests for ComfyUI nodes that use the Omni API client, with a mocked AsyncOmni and a real API server running in a background process.
These tests cover the integration between ComfyUI node and the API server, without actual model inference logic.
It ensures that
1. Changes made to the API (e.g., request and response formats) do not break the ComfyUI frontend that use it.
2. The sampling parameters are correctly passed from the node to AsyncOmni through the API layer.
"""
import multiprocessing
import time
import traceback
from collections.abc import Iterable, Sequence
from enum import StrEnum, auto
from typing import Any, NamedTuple
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import requests
import torch
from comfy_api.input import AudioInput, VideoInput
from comfyui_vllm_omni.nodes import (
VLLMOmniGenerateImage,
VLLMOmniTTS,
VLLMOmniUnderstanding,
VLLMOmniVoiceClone,
)
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams
from PIL import Image
from vllm import SamplingParams
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
class ServerCase(NamedTuple):
"""Parametrizing the model to serve."""
served_model: str
stage_list: list
stage_configs: list[dict]
outputs: list[OmniRequestOutput]
class SamplingCase(NamedTuple):
"""Parametrizing the input sampling parameters."""
kind: "SamplingKind"
sampling_params: dict | list[dict] | None
class SamplingKind(StrEnum):
IMAGE_NONE = auto()
IMAGE_DIFFUSION_SINGLE = auto()
UNDERSTANDING_NONE = auto()
UNDERSTANDING_AR_LIST = auto()
TTS_NONE = auto()
TTS_DIFFUSION_SINGLE = auto()
# Pre-defined arguments to be used in function calls during the tests
IMAGE_WIDTH = 64
IMAGE_HEIGHT = 64
DIFFUSION_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams(
{
"n": 2,
"num_inference_steps": 30,
"guidance_scale": 6.0,
"true_cfg_scale": 1.5,
}
)
AR_LIST_SAMPLING_PARAMS = [
AutoregressionSamplingParams(
{
"max_tokens": 64,
"temperature": 0.6,
"top_p": 0.9,
"repetition_penalty": 1.0,
"seed": 21,
}
),
AutoregressionSamplingParams(
{
"max_tokens": 96,
"temperature": 0.75,
"top_p": 0.85,
"repetition_penalty": 1.05,
"seed": 22,
}
),
AutoregressionSamplingParams(
{
"max_tokens": 128,
"temperature": 0.8,
"top_p": 0.8,
"repetition_penalty": 1.1,
"seed": 23,
}
),
]
def _build_image_output(size: tuple[int, int] = (IMAGE_WIDTH, IMAGE_HEIGHT), color: str = "red") -> Image.Image:
return Image.new("RGB", size, color=color)
def _build_text_output(text: str = "This is a test response.") -> OmniRequestOutput:
completion_output = CompletionOutput(
index=0,
text=text,
token_ids=[1, 2, 3],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
request_output = RequestOutput(
request_id="test_req_text",
prompt="test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
)
return OmniRequestOutput(
request_id="test_req_text",
finished=True,
final_output_type="text",
request_output=request_output,
)
def _build_audio_chat_output(num_samples: int = 24000) -> OmniRequestOutput:
completion_output = CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=0.0,
logprobs=None,
finish_reason="stop",
stop_reason=None,
)
completion_output.multimodal_output = {"audio": [torch.zeros(1, num_samples)]}
request_output = RequestOutput(
request_id="test_req_audio_chat",
prompt="test prompt",
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[completion_output],
finished=True,
metrics=None,
lora_request=None,
)
return OmniRequestOutput(
request_id="test_req_audio_chat",
finished=True,
final_output_type="audio",
request_output=request_output,
)
def _build_audio_speech_output(num_samples: int = 24000) -> OmniRequestOutput:
return OmniRequestOutput.from_diffusion(
request_id="test_req_audio_speech",
images=[],
multimodal_output={"audio": torch.zeros(num_samples), "sr": 24000},
final_output_type="audio",
)
def _build_diffusion_image_output_for_images_endpoint() -> OmniRequestOutput:
return OmniRequestOutput.from_diffusion(
request_id="test_req_img_dalle",
images=[_build_image_output()],
final_output_type="image",
)
def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
request_output = MagicMock()
request_output.images = [_build_image_output(color="blue")]
request_output.finished = True
return OmniRequestOutput(
request_id="test_req_img_chat",
finished=True,
final_output_type="image",
request_output=request_output,
)
def _assert_sampling_param_values(received: OmniSamplingParams, expected: dict[str, Any]):
for key, expected_value in expected.items():
actual_value = getattr(received, key, None)
assert actual_value == expected_value, (
f"Expected sampling param '{key}'={expected_value}, got {actual_value}. The received sampling params: {received}"
)
def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: SamplingCase, server_case: ServerCase):
async def _mock_generate(*args, **kwargs):
received_sampling_params_list: Sequence[OmniSamplingParams] | None = (
args[2] if len(args) > 2 else kwargs.get("sampling_params_list")
)
assert received_sampling_params_list is not None, (
"In the current codebase, the API layer always provides not-None sampling parameter list when calling `AsyncOmni.generate`"
"This test also uses this assumption for now."
"If this assertion fails, it means the API layer has changed and this test needs to be updated accordingly."
"It does not necessarily mean there is a bug, because `AsyncOmni.generate` does allow sampling_params_list to be None."
)
assert isinstance(received_sampling_params_list, Sequence), "sampling_params_list should be a Sequence"
if sampling_case.kind is SamplingKind.IMAGE_NONE:
assert len(received_sampling_params_list) == 1
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
},
)
elif sampling_case.kind is SamplingKind.IMAGE_DIFFUSION_SINGLE:
assert len(received_sampling_params_list) == 1
expected = DIFFUSION_SINGLE_SAMPLING_PARAMS.copy()
expected["num_outputs_per_prompt"] = expected.pop("n") # convert from n to num_outputs_per_prompt
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
**expected,
},
)
elif sampling_case.kind is SamplingKind.UNDERSTANDING_NONE:
assert len(received_sampling_params_list) == 3
elif sampling_case.kind is SamplingKind.UNDERSTANDING_AR_LIST:
assert len(received_sampling_params_list) == 3
for i, expected in enumerate(AR_LIST_SAMPLING_PARAMS):
_assert_sampling_param_values(received_sampling_params_list[i], expected)
elif sampling_case.kind in {SamplingKind.TTS_NONE, SamplingKind.TTS_DIFFUSION_SINGLE}:
assert len(received_sampling_params_list) == 1
else:
raise AssertionError(f"Unknown sampling case: {sampling_case.kind}")
for output in outputs:
yield output
return _mock_generate
@pytest.fixture
def server_case(request) -> ServerCase:
return request.param
@pytest.fixture
def sampling_case(request) -> SamplingCase:
return request.param
@pytest.fixture
def mock_async_omni(server_case: ServerCase, sampling_case: SamplingCase):
async def _mock_preprocess_chat(self, *args, **kwargs):
return ([{"role": "user", "content": "test"}], [{"prompt": "test prompt"}])
# Need to mock AsyncOmni itself (not only its generate method) because
# 1. The API layer uses its stage_list and stage_configs attributes
# 2. Its __init__ method has slow side effects (model & config loading).
with (
patch("vllm_omni.entrypoints.openai.api_server.AsyncOmni") as MockAsyncOmni,
patch(
"vllm_omni.entrypoints.openai.serving_chat.OmniOpenAIServingChat._preprocess_chat",
new=_mock_preprocess_chat,
),
):
mock_instance = AsyncMock()
mock_instance.generate = _build_mock_outputs(server_case.outputs, sampling_case, server_case)
mock_instance.stage_list = server_case.stage_list
mock_instance.stage_configs = server_case.stage_configs
mock_instance.default_sampling_params_list = [
SamplingParams() if stage.get("stage_type") != "diffusion" else MagicMock()
for stage in server_case.stage_configs
]
mock_instance.errored = False
mock_instance.dead_error = RuntimeError("Mock engine error")
mock_instance.model_config = MagicMock(max_model_len=4096, io_processor_plugin=None)
mock_instance.io_processor = MagicMock()
mock_instance.input_processor = MagicMock()
mock_instance.shutdown = MagicMock()
mock_instance.get_vllm_config = AsyncMock(return_value=None)
mock_instance.get_supported_tasks = AsyncMock(return_value=["generate"])
mock_instance.get_tokenizer = AsyncMock(return_value=None)
MockAsyncOmni.return_value = mock_instance
yield MockAsyncOmni
@pytest.fixture
def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni):
"""Set up a API server in background process from command line with parametrized model name and mocked AsyncOmni."""
parser = FlexibleArgumentParser()
subparsers = parser.add_subparsers(dest="command")
cmd = OmniServeCommand()
cmd.subparser_init(subparsers)
port = unused_tcp_port_factory()
args = parser.parse_args(["serve", server_case.served_model, "--omni", "--port", str(port)])
def run_server():
try:
cmd.cmd(args)
except Exception:
traceback.print_exc()
server_process = multiprocessing.Process(target=run_server)
server_process.start()
# Wait for the server to be ready by polling the health endpoint.
wait_time = 30
wait_poll_interval = 1
for _ in range(wait_time // wait_poll_interval):
try:
response = requests.get(f"http://127.0.0.1:{port}/health")
if response.status_code == 200:
break
except requests.ConnectionError:
time.sleep(wait_poll_interval)
else:
if server_process.is_alive():
server_process.terminate()
server_process.join(timeout=5)
if server_process.is_alive():
server_process.kill()
server_process.join(timeout=5)
pytest.fail(f"API server failed to start within {wait_time} seconds")
yield f"http://127.0.0.1:{port}/v1"
if server_process.is_alive():
server_process.terminate()
server_process.join(timeout=10)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,model,image_input",
[
pytest.param(
ServerCase(
served_model="Tongyi-MAI/Z-Image-Turbo",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_image_output_for_images_endpoint()],
),
"Tongyi-MAI/Z-Image-Turbo",
False,
id="text-to-image-dalle-endpoint",
),
pytest.param(
ServerCase(
served_model="ByteDance-Seed/BAGEL-7B-MoT",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_image_output_for_chat_endpoint()],
),
"ByteDance-Seed/BAGEL-7B-MoT",
False,
id="text-to-image-bagel-chat-endpoint",
),
pytest.param(
ServerCase(
served_model="Qwen/Qwen-Image-Edit",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_image_output_for_images_endpoint()],
),
"Qwen/Qwen-Image-Edit",
True,
id="image-to-image-dalle-endpoint",
),
pytest.param(
ServerCase(
served_model="ByteDance-Seed/BAGEL-7B-MoT",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_image_output_for_chat_endpoint()],
),
"ByteDance-Seed/BAGEL-7B-MoT",
True,
id="image-to-image-bagel-chat-endpoint",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.IMAGE_DIFFUSION_SINGLE, sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_image_generation_node(api_server: str, model: str, image_input: bool, sampling_case: SamplingCase):
node = VLLMOmniGenerateImage()
kwargs = {
"url": api_server,
"model": model,
"prompt": "A beautiful sunset",
"width": IMAGE_WIDTH,
"height": IMAGE_HEIGHT,
}
if image_input:
kwargs["image"] = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
if sampling_case.sampling_params is not None:
kwargs["sampling_params"] = sampling_case.sampling_params
print(f"!!!!!! Calling {model} node.generate with kwargs: {sampling_case.sampling_params}")
result = await node.generate(**kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], torch.Tensor)
assert result[0].shape == (1, IMAGE_WIDTH, IMAGE_HEIGHT, 3)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case",
[
pytest.param(
ServerCase(
served_model="Qwen/Qwen2.5-Omni-7B",
stage_list=[
MagicMock(is_comprehension=True, model_stage="llm"),
MagicMock(is_comprehension=False, model_stage="llm"),
MagicMock(is_comprehension=False, model_stage="llm"),
],
stage_configs=[
{"stage_type": "llm"},
{"stage_type": "llm"},
{"stage_type": "llm"},
],
outputs=[_build_audio_chat_output(), _build_text_output("Understanding response")],
),
id="multimodal-understanding",
)
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.UNDERSTANDING_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.UNDERSTANDING_AR_LIST, sampling_params=AR_LIST_SAMPLING_PARAMS),
id="ar-sampling-params-list",
),
],
indirect=["sampling_case"],
)
async def test_understanding_node(api_server: str, sampling_case: SamplingCase):
node = VLLMOmniUnderstanding()
image = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
video = VideoInput(b"mock_video_for_test") # type: ignore[reportAbstractUsage]
audio: AudioInput = {"waveform": torch.zeros((1, 1, 24000), dtype=torch.float32), "sample_rate": 24000}
text_response, audio_response = await node.generate(
url=api_server,
model="Qwen/Qwen2.5-Omni-7B",
prompt="Describe all modalities.",
image=image,
audio=audio,
video=video,
sampling_params=sampling_case.sampling_params,
output_text=True,
output_audio=True,
use_audio_in_video=True,
)
assert text_response == "Understanding response"
assert isinstance(audio_response, dict)
assert audio_response["sample_rate"] == 24000
assert isinstance(audio_response["waveform"], torch.Tensor)
assert audio_response["waveform"].shape == (1, 1, 24000)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,node_cls,call_kwargs",
[
pytest.param(
ServerCase(
served_model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
stage_list=["llm"],
stage_configs=[{"stage_type": "llm"}],
outputs=[_build_audio_speech_output()],
),
VLLMOmniTTS,
{
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
"input": "Hello from TTS test",
"voice": "Vivian",
"response_format": "wav",
"speed": 1.0,
"model_specific_params": None,
},
id="tts",
),
pytest.param(
ServerCase(
served_model="Qwen/Qwen3-TTS-12Hz-1.7B-Base",
stage_list=["llm"],
stage_configs=[{"stage_type": "llm"}],
outputs=[_build_audio_speech_output()],
),
VLLMOmniVoiceClone,
{
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
"input": "Hello from voice clone test",
"voice": "Vivian",
"response_format": "wav",
"speed": 1.0,
"ref_audio": {"waveform": torch.zeros((1, 1, 24000), dtype=torch.float32), "sample_rate": 24000},
"ref_text": "Reference transcript",
"x_vector_only_mode": False,
"model_specific_params": None,
},
id="tts-voice-clone",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.TTS_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.TTS_DIFFUSION_SINGLE, sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_tts_nodes(api_server: str, node_cls, call_kwargs: dict, sampling_case: SamplingCase):
node = node_cls()
actual_kwargs = dict(call_kwargs)
if sampling_case.sampling_params is not None:
actual_kwargs["model_specific_params"] = sampling_case.sampling_params
result = await node.generate(url=api_server, **actual_kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], dict)
assert result[0]["sample_rate"] == 24000
assert isinstance(result[0]["waveform"], torch.Tensor)
assert result[0]["waveform"].shape == (1, 1, 24000)
+9 -6
View File
@@ -15,6 +15,9 @@ import torch
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl
from vllm_omni.platforms import current_omni_platform
is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu()
def create_attention_mask(batch_size: int, seq_len: int, valid_len: int, device: torch.device) -> torch.Tensor:
@@ -56,7 +59,7 @@ def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0
return torch.cat([tensor, padding], dim=1)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
def test_padding_equivalence():
"""
Case 1: Test that padded and unpadded inputs produce similar outputs.
@@ -68,7 +71,7 @@ def test_padding_equivalence():
Expected: Output A and Output B should be very close.
"""
device = torch.device("cuda")
device = torch.device(current_omni_platform.device_type)
dtype = torch.bfloat16
# Configuration
@@ -149,7 +152,7 @@ def test_padding_equivalence():
print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
def test_fa_vs_sdpa():
"""
Case 2: Compare FlashAttention and SDPA backends with padding.
@@ -162,7 +165,7 @@ def test_fa_vs_sdpa():
Expected: FA and SDPA outputs should be very close.
"""
device = torch.device("cuda")
device = torch.device(current_omni_platform.device_type)
dtype = torch.bfloat16
# Configuration
@@ -265,8 +268,8 @@ if __name__ == "__main__":
print("Running FlashAttention Padding Tests...")
print("=" * 60)
# Try to run CUDA tests
if torch.cuda.is_available():
# Try to run tests
if is_gpu:
try:
print("\n[Running Case 1: Padding Equivalence for FA]")
test_padding_equivalence()
+225
View File
@@ -0,0 +1,225 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for HSDP (Hybrid Sharded Data Parallel) configuration and utilities.
These tests verify HSDP configuration logic without requiring a distributed environment.
"""
import pytest
import torch.nn as nn
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig
pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu]
class TestHSDPInferenceConfig:
"""Tests for HSDPInferenceConfig dataclass."""
def test_default_values(self):
"""Test default configuration values."""
config = HSDPInferenceConfig()
assert config.enabled is False
assert config.hsdp_replicate_size == 1
assert config.hsdp_shard_size == -1
def test_custom_values(self):
"""Test custom configuration values."""
config = HSDPInferenceConfig(
enabled=True,
hsdp_replicate_size=2,
hsdp_shard_size=4,
)
assert config.enabled is True
assert config.hsdp_replicate_size == 2
assert config.hsdp_shard_size == 4
class TestDiffusionParallelConfigHSDP:
"""Tests for HSDP settings in DiffusionParallelConfig."""
def test_hsdp_disabled_by_default(self):
"""HSDP should be disabled by default."""
config = DiffusionParallelConfig()
assert config.use_hsdp is False
assert config.hsdp_shard_size == -1
assert config.hsdp_replicate_size == 1
def test_hsdp_auto_shard_size(self):
"""Test auto-calculation of hsdp_shard_size when use_hsdp=True."""
# ulysses_degree=4 -> world_size=4
# hsdp_shard_size should be auto-calculated as 4 // 1 = 4
config = DiffusionParallelConfig(
ulysses_degree=4,
use_hsdp=True,
)
assert config.world_size == 4
assert config.hsdp_shard_size == 4
assert config.hsdp_replicate_size == 1
def test_hsdp_auto_shard_size_fails_standalone(self):
"""Test that auto-calculate fails when other parallelism is all 1."""
# When all other parallelism is 1, cannot auto-calculate
# User must specify hsdp_shard_size explicitly
with pytest.raises(ValueError, match="Cannot auto-calculate hsdp_shard_size"):
DiffusionParallelConfig(
use_hsdp=True,
# All other parallelism defaults to 1
)
def test_hsdp_standalone_mode(self):
"""Test standalone HSDP (HSDP without other parallelism)."""
# Standalone HSDP: all other parallelism=1, explicit shard_size
config = DiffusionParallelConfig(
use_hsdp=True,
hsdp_shard_size=4, # Explicit shard size
hsdp_replicate_size=1,
)
# world_size should be determined by HSDP
assert config.world_size == 4
assert config.hsdp_shard_size == 4
assert config.hsdp_replicate_size == 1
def test_hsdp_standalone_with_replicate(self):
"""Test standalone HSDP with replication."""
config = DiffusionParallelConfig(
use_hsdp=True,
hsdp_shard_size=4,
hsdp_replicate_size=2,
)
# world_size = shard_size * replicate_size
assert config.world_size == 8
assert config.hsdp_shard_size == 4
assert config.hsdp_replicate_size == 2
def test_hsdp_with_replicate(self):
"""Test HSDP with replication (hybrid mode) combined with other parallelism."""
# world_size=8, replicate=2 -> shard_size should be 4
config = DiffusionParallelConfig(
ulysses_degree=8,
use_hsdp=True,
hsdp_replicate_size=2,
)
assert config.world_size == 8
assert config.hsdp_shard_size == 4
assert config.hsdp_replicate_size == 2
def test_hsdp_explicit_shard_size_valid(self):
"""Test explicit hsdp_shard_size that matches world_size."""
config = DiffusionParallelConfig(
ulysses_degree=4,
use_hsdp=True,
hsdp_shard_size=4,
hsdp_replicate_size=1,
)
assert config.hsdp_shard_size == 4
def test_hsdp_explicit_shard_size_invalid(self):
"""Test that invalid HSDP dimensions raise an error when combined with other parallelism."""
with pytest.raises(ValueError, match="HSDP dimensions"):
DiffusionParallelConfig(
ulysses_degree=4, # world_size=4
use_hsdp=True,
hsdp_shard_size=3, # 1 * 3 != 4
hsdp_replicate_size=1,
)
def test_hsdp_replicate_size_exceeds_world_size(self):
"""Test that replicate_size > world_size raises an error."""
with pytest.raises(ValueError, match="replicate_size.*must evenly divide world_size"):
DiffusionParallelConfig(
ulysses_degree=4, # world_size=4
use_hsdp=True,
hsdp_replicate_size=8, # 8 > 4, invalid
)
def test_hsdp_combined_world_size(self):
"""Test that combined HSDP matches other parallelism world_size."""
config_no_hsdp = DiffusionParallelConfig(ulysses_degree=4)
config_with_hsdp = DiffusionParallelConfig(
ulysses_degree=4,
use_hsdp=True,
hsdp_shard_size=4,
)
# When combined with other parallelism, world_size should match
assert config_no_hsdp.world_size == config_with_hsdp.world_size == 4
def test_hsdp_standalone_world_size(self):
"""Test that standalone HSDP determines world_size."""
config_hsdp = DiffusionParallelConfig(
use_hsdp=True,
hsdp_shard_size=8,
)
# Standalone HSDP: world_size is determined by HSDP
assert config_hsdp.world_size == 8
def test_hsdp_cannot_use_with_tp(self):
"""Test that HSDP and Tensor Parallelism cannot be used together."""
with pytest.raises(ValueError, match="cannot be used with Tensor Parallelism"):
DiffusionParallelConfig(
tensor_parallel_size=2,
use_hsdp=True,
hsdp_shard_size=4,
)
def test_from_dict_with_hsdp(self):
"""Test creating config from dict with HSDP settings."""
config = DiffusionParallelConfig.from_dict(
{
"ulysses_degree": 4,
"use_hsdp": True,
"hsdp_replicate_size": 2,
}
)
assert config.use_hsdp is True
assert config.hsdp_replicate_size == 2
assert config.hsdp_shard_size == 2 # auto: 4 // 2
class TestHSDPShardConditions:
"""Tests for _hsdp_shard_conditions matching logic."""
@staticmethod
def _is_transformer_block(name: str, module: nn.Module) -> bool:
"""Example shard condition matching transformer blocks."""
return "blocks" in name and name.split(".")[-1].isdigit()
def test_condition_matches_blocks(self):
"""Test that condition matches transformer block patterns."""
cond = self._is_transformer_block
# Should match
assert cond("blocks.0", nn.Linear(10, 10)) is True
assert cond("blocks.15", nn.Linear(10, 10)) is True
assert cond("transformer.blocks.0", nn.Linear(10, 10)) is True
# Should not match
assert cond("blocks", nn.Linear(10, 10)) is False
assert cond("blocks.norm", nn.Linear(10, 10)) is False
assert cond("embeddings", nn.Linear(10, 10)) is False
def test_model_with_shard_conditions(self):
"""Test model with _hsdp_shard_conditions attribute."""
class MockModel(nn.Module):
@staticmethod
def _is_block(name: str, module: nn.Module) -> bool:
return name.startswith("blocks.") and name.split(".")[-1].isdigit()
_hsdp_shard_conditions = [_is_block]
def __init__(self):
super().__init__()
self.blocks = nn.ModuleList([nn.Linear(10, 10) for _ in range(2)])
model = MockModel()
conditions = getattr(model, "_hsdp_shard_conditions", None)
assert conditions is not None
assert len(conditions) == 1
# Verify conditions work on actual model modules
matched = []
for name, module in model.named_modules():
if any(cond(name, module) for cond in conditions):
matched.append(name)
assert "blocks.0" in matched
assert "blocks.1" in matched
+182 -26
View File
@@ -38,21 +38,77 @@ class _FakeLinearBase(LinearBase):
torch.nn.Module.__init__(self)
class _DummyBaseLayerWithLoRA(torch.nn.Module):
def __init__(self, base_layer: torch.nn.Module):
super().__init__()
self.base_layer = base_layer
self.set_calls: list[
tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor]
] = []
self.reset_calls: int = 0
self.create_calls: int = 0
def set_lora(self, index: int, lora_a, lora_b):
assert index == 0
self.set_calls.append((lora_a, lora_b))
def reset_lora(self, index: int):
assert index == 0
self.reset_calls += 1
def create_lora_weights(self, max_loras, lora_config, model_config):
# Needs to be callable for scale test when rank changes, but not
# actually used since we mock everything and check everything based
# on set calls.
self.create_calls += 1
class _DummyPipeline(torch.nn.Module):
def __init__(self):
super().__init__()
self.transformer = torch.nn.Module()
self.transformer.foo = _FakeLinearBase()
class _DummyLM(torch.nn.Module):
"""LoRA enabled wrapper for _DummyPipeline."""
def __init__(self, rank: int):
super().__init__()
self.transformer = torch.nn.Module()
self.transformer.foo = _DummyBaseLayerWithLoRA(_FakeLinearBase())
self.rank = rank
self.loras = self.get_lora_modules()
def get_lora_modules(self):
return {"transformer.foo": self._get_initial_lora(self.rank)}
def get_lora(self, k: str) -> LoRALayerWeights:
"""Get the unscaled LoRA weights for transformer.foo"""
return self.loras[k]
def _get_initial_lora(self, rank: int) -> LoRALayerWeights:
"""Initializes a dummy LoRA for the current rank."""
A = torch.ones((rank, 4))
B = torch.ones((4, rank))
return LoRALayerWeights(
module_name="foo",
rank=rank,
lora_alpha=rank,
lora_a=A,
lora_b=B,
)
def test_lora_manager_supported_modules_are_stable_with_wrapped_layers(monkeypatch):
# Simulate a pipeline that already contains LoRA wrappers where the original
# LinearBase is nested under ".base_layer".
import vllm_omni.diffusion.lora.manager as manager_mod
class _DummyBaseLayerWithLoRA(torch.nn.Module):
def __init__(self, base_layer: torch.nn.Module):
super().__init__()
self.base_layer = base_layer
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
pipeline = torch.nn.Module()
pipeline.transformer = torch.nn.Module()
pipeline.transformer.foo = _DummyBaseLayerWithLoRA(_FakeLinearBase())
pipeline = _DummyLM(rank=2)
# vLLM helper would see only the nested LinearBase and yield "base_layer".
assert get_supported_lora_modules(pipeline) == ["base_layer"]
@@ -71,11 +127,6 @@ def test_lora_manager_supported_modules_are_stable_with_wrapped_layers(monkeypat
def test_lora_manager_replace_layers_does_not_rewrap_base_layer(monkeypatch):
import vllm_omni.diffusion.lora.manager as manager_mod
class _DummyBaseLayerWithLoRA(torch.nn.Module):
def __init__(self, base_layer: torch.nn.Module):
super().__init__()
self.base_layer = base_layer
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs):
@@ -116,11 +167,6 @@ def test_lora_manager_replace_layers_does_not_rewrap_base_layer(monkeypatch):
def test_lora_manager_replaces_packed_layer_when_targeting_sublayers(monkeypatch):
import vllm_omni.diffusion.lora.manager as manager_mod
class _DummyBaseLayerWithLoRA(torch.nn.Module):
def __init__(self, base_layer: torch.nn.Module):
super().__init__()
self.base_layer = base_layer
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs):
@@ -189,9 +235,8 @@ def test_lora_manager_activates_fused_lora_on_packed_layer():
},
)()
}
manager._adapter_scales = {7: 0.5}
manager._activate_adapter(7)
manager._activate_adapter(7, 0.5)
assert packed_layer.reset_calls == 0
assert len(packed_layer.set_calls) == 1
@@ -234,9 +279,8 @@ def test_lora_manager_activates_packed_lora_from_sublayers():
manager._registered_adapters = {
1: type("LM", (), {"id": 1, "loras": loras, "get_lora": lambda self, k: self.loras.get(k)})()
}
manager._adapter_scales = {1: 2.0}
manager._activate_adapter(1)
manager._activate_adapter(1, scale=2.0)
assert packed_layer.reset_calls == 0
assert len(packed_layer.set_calls) == 1
@@ -274,7 +318,7 @@ def test_lora_manager_evicts_lru_adapter_when_cache_full(monkeypatch):
monkeypatch.setattr(manager, "_load_adapter", _fake_load)
monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id, scale: None)
req1 = _dummy_lora_request(1)
req2 = _dummy_lora_request(2)
@@ -306,7 +350,7 @@ def test_lora_manager_does_not_evict_pinned_adapter(monkeypatch):
monkeypatch.setattr(manager, "_load_adapter", _fake_load)
monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id, scale: None)
manager.set_active_adapter(_dummy_lora_request(1), lora_scale=1.0)
assert manager.pin_adapter(1)
@@ -332,7 +376,7 @@ def test_lora_manager_warns_when_all_adapters_pinned(monkeypatch):
monkeypatch.setattr(manager, "_load_adapter", _fake_load)
monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None)
monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id, scale: None)
manager.set_active_adapter(_dummy_lora_request(1), lora_scale=1.0)
manager.set_active_adapter(_dummy_lora_request(2), lora_scale=1.0)
@@ -341,6 +385,118 @@ def test_lora_manager_warns_when_all_adapters_pinned(monkeypatch):
assert manager.pin_adapter(2)
manager.max_cached_adapters = 1
manager._evict_if_needed()
manager._evict_for_new_adapter()
assert set(manager.list_adapters()) == {1, 2}
def test_lora_manager_applies_multiple_scales_correctly(monkeypatch):
"""Ensure that the LoRA manager applies scales correctly when the
active adapter receives a different scale, i.e., the rank is unchanged.
"""
import vllm_omni.diffusion.lora.manager as manager_mod
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyLoRALayer)
rank = 2
adapter_id = 7
req1 = _dummy_lora_request(adapter_id)
scale_1 = 0.25
scale_2 = 0.5
lora_model = _DummyLM(rank=rank)
manager = DiffusionLoRAManager(
pipeline=_DummyPipeline(),
device=torch.device("cpu"),
dtype=torch.bfloat16,
)
def _fake_load(_req: LoRARequest):
peft_helper = type("PH", (), {"r": rank})()
return lora_model, peft_helper
monkeypatch.setattr(manager, "_load_adapter", _fake_load)
manager._registered_adapters = {
adapter_id: lora_model,
}
manager._lora_modules = {"transformer.foo": lora_model.transformer.foo}
# After the first scale, all B values should go from 1 -> scale_1
manager.set_active_adapter(req1, lora_scale=scale_1)
assert len(lora_model.transformer.foo.set_calls) == 1
lora_a, lora_b = lora_model.transformer.foo.set_calls[0]
assert torch.all(lora_a == 1)
assert torch.all(lora_b == scale_1)
# After the second scale, all B values should go from 1 -> scale_2
manager.set_active_adapter(req1, lora_scale=scale_2)
assert len(lora_model.transformer.foo.set_calls) == 2
lora_a, lora_b = lora_model.transformer.foo.set_calls[1]
assert torch.all(lora_a == 1)
assert torch.all(lora_b == scale_2)
def test_lora_manager_scales_correctly_with_rank_changes(monkeypatch):
"""Ensure that the LoRA manager correctly handles scaling when the rank
is changed and the buffers are reset + we reactivate.
"""
import vllm_omni.diffusion.lora.manager as manager_mod
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
rank = 2
adapter_id = 7
req1 = _dummy_lora_request(adapter_id)
initial_scale = 0.5
lora_model = _DummyLM(rank=rank)
manager = DiffusionLoRAManager(
pipeline=_DummyPipeline(),
device=torch.device("cpu"),
dtype=torch.bfloat16,
)
def _fake_load(_req: LoRARequest):
peft_helper = type("PH", (), {"r": rank})()
return lora_model, peft_helper
monkeypatch.setattr(manager, "_load_adapter", _fake_load)
manager._registered_adapters = {
adapter_id: lora_model,
}
manager._lora_modules = {"transformer.foo": lora_model.transformer.foo}
# Activate adapter with initial scale
manager.set_active_adapter(req1, lora_scale=initial_scale)
assert lora_model.transformer.foo.create_calls == 0
assert len(lora_model.transformer.foo.set_calls) == 1
lora_a, lora_b = lora_model.transformer.foo.set_calls[0]
assert torch.all(lora_a == 1)
assert torch.all(lora_b == initial_scale)
# Increase the rank; this resets the buffers, so the adapter is activated again
manager._ensure_max_lora_rank(8)
# Ensure we actually took the rank expansion path, which recreates
# and sets the weight buffets, but that the scale didn't change
assert lora_model.transformer.foo.create_calls == 1
assert len(lora_model.transformer.foo.set_calls) == 2
lora_a, lora_b = lora_model.transformer.foo.set_calls[1]
assert torch.all(lora_a == 1)
assert torch.all(lora_b == initial_scale)
def test_scale_keys_are_rounded():
"""Ensure that added adapter scales are rounded to avoid lookup
issues due to precision differences, e.g., computed scales.
"""
manager = DiffusionLoRAManager(
pipeline=_DummyPipeline(),
device=torch.device("cpu"),
dtype=torch.bfloat16,
)
adapter_id = 1
# Currently we round keys to 3 decimal places
manager._update_adapter_scale(adapter_id, 0.0031)
assert manager._adapter_scales[adapter_id] == 0.003
+73
View File
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn as nn
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
class _DummyPipelineModel(nn.Module):
def __init__(self, *, source_prefix: str):
super().__init__()
self.transformer = nn.Linear(2, 2, bias=False)
self.vae = nn.Linear(2, 2, bias=False)
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path="dummy",
subfolder="transformer",
revision=None,
prefix=source_prefix,
fall_back_to_pt=True,
)
]
def load_weights(self, weights):
params = dict(self.named_parameters())
loaded: set[str] = set()
for name, tensor in weights:
if name not in params:
continue
params[name].data.copy_(tensor.to(dtype=params[name].dtype))
loaded.add(name)
return loaded
def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoader:
loader = object.__new__(DiffusersPipelineLoader)
loader.counter_before_loading_weights = 0.0
loader.counter_after_loading_weights = 0.0
def _iter_weights(_model):
for name in weight_names:
yield name, torch.zeros((2, 2))
loader.get_all_weights = _iter_weights # type: ignore[assignment]
return loader
def test_strict_check_only_validates_source_prefix_parameters():
model = _DummyPipelineModel(source_prefix="transformer.")
loader = _make_loader_with_weights(["transformer.weight"])
# Should not require VAE parameters because they are outside weights_sources.
loader.load_weights(model)
def test_strict_check_raises_when_source_parameters_are_missing():
model = _DummyPipelineModel(source_prefix="transformer.")
loader = _make_loader_with_weights([])
with pytest.raises(ValueError, match="transformer.weight"):
loader.load_weights(model)
def test_empty_source_prefix_keeps_full_model_strict_check():
model = _DummyPipelineModel(source_prefix="")
loader = _make_loader_with_weights(["transformer.weight"])
with pytest.raises(ValueError, match="vae.weight"):
loader.load_weights(model)
@@ -54,6 +54,7 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar
runner.od_config = SimpleNamespace(
cache_backend=cache_backend_name,
enable_cache_dit_summary=enable_cache_dit_summary,
parallel_config=SimpleNamespace(use_hsdp=False),
)
runner.kv_transfer_manager = SimpleNamespace(receive_kv_cache=lambda req, target_device: None)
return runner
+78 -83
View File
@@ -10,10 +10,9 @@ This module tests the DiffusionWorker implementation:
- wake_up: Waking worker from sleep mode
"""
from unittest.mock import Mock, patch
import pytest
import torch
from pytest_mock import MockerFixture
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
@@ -21,9 +20,9 @@ pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@pytest.fixture
def mock_od_config():
def mock_od_config(mocker: MockerFixture):
"""Create a mock OmniDiffusionConfig."""
config = Mock()
config = mocker.Mock()
config.num_gpus = 1
config.master_port = 12345
config.enable_sleep_mode = False
@@ -34,22 +33,23 @@ def mock_od_config():
@pytest.fixture
def mock_gpu_worker(mock_od_config):
def mock_gpu_worker(mocker: MockerFixture, mock_od_config):
"""Create a DiffusionWorker with mocked initialization."""
with patch.object(DiffusionWorker, "init_device"), patch.object(DiffusionWorker, "load_model"):
worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config)
# Mock the model_runner with pipeline
worker.model_runner = Mock()
worker.model_runner.pipeline = Mock()
worker.device = torch.device("cuda", 0)
worker._sleep_saved_buffers = {}
return worker
mocker.patch.object(DiffusionWorker, "init_device")
mocker.patch.object(DiffusionWorker, "load_model")
worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config)
# Mock the model_runner with pipeline
worker.model_runner = mocker.Mock()
worker.model_runner.pipeline = mocker.Mock()
worker.device = torch.device("cuda", 0)
worker._sleep_saved_buffers = {}
return worker
class TestDiffusionWorkerLoadWeights:
"""Test DiffusionWorker.load_weights method."""
def test_load_weights_calls_pipeline(self, mock_gpu_worker):
def test_load_weights_calls_pipeline(self, mocker: MockerFixture, mock_gpu_worker):
"""Test that load_weights delegates to model_runner.load_weights."""
# Setup mock weights
mock_weights = [
@@ -59,7 +59,7 @@ class TestDiffusionWorkerLoadWeights:
expected_loaded = {"layer1.weight", "layer2.weight"}
# Configure model_runner mock
mock_gpu_worker.model_runner.load_weights = Mock(return_value=expected_loaded)
mock_gpu_worker.model_runner.load_weights = mocker.Mock(return_value=expected_loaded)
# Call load_weights
result = mock_gpu_worker.load_weights(mock_weights)
@@ -68,9 +68,9 @@ class TestDiffusionWorkerLoadWeights:
mock_gpu_worker.model_runner.load_weights.assert_called_once_with(mock_weights)
assert result == expected_loaded
def test_load_weights_empty_iterable(self, mock_gpu_worker):
def test_load_weights_empty_iterable(self, mocker: MockerFixture, mock_gpu_worker):
"""Test load_weights with empty weights iterable."""
mock_gpu_worker.model_runner.load_weights = Mock(return_value=set())
mock_gpu_worker.model_runner.load_weights = mocker.Mock(return_value=set())
result = mock_gpu_worker.load_weights([])
@@ -81,11 +81,12 @@ class TestDiffusionWorkerLoadWeights:
class TestDiffusionWorkerSleep:
"""Test DiffusionWorker.sleep method."""
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_level_1(self, mock_allocator_class, mock_platform, mock_get_process_memory, mock_gpu_worker):
def test_sleep_level_1(self, mocker: MockerFixture, mock_gpu_worker):
"""Test sleep mode level 1 (offload weights only)."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
mock_get_process_memory = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
# Setup process-scoped memory mocks
# Before sleep: 3GB used
# After sleep: 1GB used (freed 2GB)
@@ -95,9 +96,9 @@ class TestDiffusionWorkerSleep:
]
# Setup allocator mock
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.sleep = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.sleep = mocker.Mock()
# Call sleep with level 1
result = mock_gpu_worker.sleep(level=1)
@@ -108,11 +109,12 @@ class TestDiffusionWorkerSleep:
# Verify buffers were NOT saved (level 1 doesn't save buffers)
assert len(mock_gpu_worker._sleep_saved_buffers) == 0
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_level_2(self, mock_allocator_class, mock_platform, mock_get_process_memory, mock_gpu_worker):
def test_sleep_level_2(self, mocker: MockerFixture, mock_gpu_worker):
"""Test sleep mode level 2 (offload all, save buffers)."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
mock_get_process_memory = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
# Setup process-scoped memory mocks
mock_get_process_memory.side_effect = [
5 * 1024**3, # Before sleep
@@ -120,14 +122,14 @@ class TestDiffusionWorkerSleep:
]
# Setup allocator mock
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.sleep = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.sleep = mocker.Mock()
# Mock pipeline buffers
mock_buffer1 = torch.randn(10, 10)
mock_buffer2 = torch.randn(20, 20)
mock_gpu_worker.model_runner.pipeline.named_buffers = Mock(
mock_gpu_worker.model_runner.pipeline.named_buffers = mocker.Mock(
return_value=[
("buffer1", mock_buffer1),
("buffer2", mock_buffer2),
@@ -146,42 +148,32 @@ class TestDiffusionWorkerSleep:
assert "buffer1" in mock_gpu_worker._sleep_saved_buffers
assert "buffer2" in mock_gpu_worker._sleep_saved_buffers
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_memory_freed_validation(
self,
mock_allocator_class,
mock_platform,
mock_get_process_memory,
mock_gpu_worker,
):
def test_sleep_memory_freed_validation(self, mocker: MockerFixture, mock_gpu_worker):
"""Test that sleep validates memory was actually freed."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
mock_get_process_memory = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
# Simulate process memory increase (should trigger assertion error)
mock_get_process_memory.side_effect = [
1 * 1024**3, # Before sleep: 1GB used
3 * 1024**3, # After sleep: 3GB used (negative freed)
]
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.sleep = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.sleep = mocker.Mock()
# This should raise an assertion error
with pytest.raises(AssertionError, match="Memory usage increased after sleeping"):
mock_gpu_worker.sleep(level=1)
@patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
@patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_sleep_falls_back_to_device_memory_when_nvml_unavailable(
self,
mock_allocator_class,
mock_platform,
mock_get_process_memory,
mock_gpu_worker,
):
def test_sleep_falls_back_to_device_memory_when_nvml_unavailable(self, mocker: MockerFixture, mock_gpu_worker):
"""Test sleep uses device-scoped fallback when NVML is unavailable."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
mock_platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
mock_get_process_memory = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.get_process_gpu_memory")
mock_get_process_memory.side_effect = [None, None]
mock_platform.get_free_memory.side_effect = [
1 * 1024**3, # Before sleep
@@ -189,9 +181,9 @@ class TestDiffusionWorkerSleep:
]
mock_platform.get_device_total_memory.return_value = 8 * 1024**3
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.sleep = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.sleep = mocker.Mock()
result = mock_gpu_worker.sleep(level=1)
@@ -202,13 +194,14 @@ class TestDiffusionWorkerSleep:
class TestDiffusionWorkerWakeUp:
"""Test DiffusionWorker.wake_up method."""
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_wake_up_without_buffers(self, mock_allocator_class, mock_gpu_worker):
def test_wake_up_without_buffers(self, mocker: MockerFixture, mock_gpu_worker):
"""Test wake_up without saved buffers (level 1 sleep)."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
# Setup allocator mock
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.wake_up = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.wake_up = mocker.Mock()
# Ensure no saved buffers
mock_gpu_worker._sleep_saved_buffers = {}
@@ -220,13 +213,14 @@ class TestDiffusionWorkerWakeUp:
mock_allocator.wake_up.assert_called_once_with(["weights"])
assert result is True
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_wake_up_with_buffers(self, mock_allocator_class, mock_gpu_worker):
def test_wake_up_with_buffers(self, mocker: MockerFixture, mock_gpu_worker):
"""Test wake_up with saved buffers (level 2 sleep)."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
# Setup allocator mock
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.wake_up = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.wake_up = mocker.Mock()
# Create saved buffers
saved_buffer1 = torch.randn(10, 10)
@@ -237,12 +231,12 @@ class TestDiffusionWorkerWakeUp:
}
# Mock pipeline buffers (these will be restored)
mock_buffer1 = Mock()
mock_buffer1.data = Mock()
mock_buffer2 = Mock()
mock_buffer2.data = Mock()
mock_buffer1 = mocker.Mock()
mock_buffer1.data = mocker.Mock()
mock_buffer2 = mocker.Mock()
mock_buffer2.data = mocker.Mock()
mock_gpu_worker.model_runner.pipeline.named_buffers = Mock(
mock_gpu_worker.model_runner.pipeline.named_buffers = mocker.Mock(
return_value=[
("buffer1", mock_buffer1),
("buffer2", mock_buffer2),
@@ -263,13 +257,14 @@ class TestDiffusionWorkerWakeUp:
assert len(mock_gpu_worker._sleep_saved_buffers) == 0
assert result is True
@patch("vllm.device_allocator.cumem.CuMemAllocator")
def test_wake_up_partial_buffer_restore(self, mock_allocator_class, mock_gpu_worker):
def test_wake_up_partial_buffer_restore(self, mocker: MockerFixture, mock_gpu_worker):
"""Test wake_up only restores buffers that were saved."""
mock_allocator_class = mocker.patch("vllm.device_allocator.cumem.CuMemAllocator")
# Setup allocator mock
mock_allocator = Mock()
mock_allocator_class.get_instance = Mock(return_value=mock_allocator)
mock_allocator.wake_up = Mock()
mock_allocator = mocker.Mock()
mock_allocator_class.get_instance = mocker.Mock(return_value=mock_allocator)
mock_allocator.wake_up = mocker.Mock()
# Only save buffer1, not buffer2
saved_buffer1 = torch.randn(10, 10)
@@ -278,12 +273,12 @@ class TestDiffusionWorkerWakeUp:
}
# Mock pipeline has both buffers
mock_buffer1 = Mock()
mock_buffer1.data = Mock()
mock_buffer2 = Mock()
mock_buffer2.data = Mock()
mock_buffer1 = mocker.Mock()
mock_buffer1.data = mocker.Mock()
mock_buffer2 = mocker.Mock()
mock_buffer2.data = mocker.Mock()
mock_gpu_worker.model_runner.pipeline.named_buffers = Mock(
mock_gpu_worker.model_runner.pipeline.named_buffers = mocker.Mock(
return_value=[
("buffer1", mock_buffer1),
("buffer2", mock_buffer2),
@@ -0,0 +1,442 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import queue
import threading
from unittest.mock import Mock, patch
import pytest
import torch
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor
from vllm_omni.diffusion.scheduler import Scheduler
pytestmark = [pytest.mark.diffusion]
# ───────────────────────────────────────────── helpers ─────────────────────
def _tagged_output(tag: str) -> DiffusionOutput:
"""Return a ``DiffusionOutput`` identifiable by its *error* field."""
return DiffusionOutput(output=torch.tensor([0]), error=tag)
def _mock_request(tag: str) -> Mock:
"""Return a mock ``OmniDiffusionRequest`` identifiable by *tag*."""
req = Mock()
req.request_ids = [tag]
return req
def _make_scheduler():
"""Create a ``Scheduler`` whose *mq* / *result_mq* are backed by
plain ``queue.Queue`` objects (thread-safe, no real IPC).
Returns ``(scheduler, request_queue, result_queue)``.
"""
sched = Scheduler()
sched.num_workers = 1
sched._lock = threading.Lock()
req_q: queue.Queue = queue.Queue()
res_q: queue.Queue = queue.Queue()
mock_mq = Mock()
mock_mq.enqueue = req_q.put
mock_rmq = Mock()
mock_rmq.dequeue = lambda timeout=None: res_q.get(timeout=timeout if timeout else 10)
sched.mq = mock_mq
sched.result_mq = mock_rmq
return sched, req_q, res_q
def _make_executor(scheduler):
"""Create a ``MultiprocDiffusionExecutor`` wired to *scheduler*
without launching real worker processes.
"""
od_cfg = Mock()
od_cfg.num_gpus = 1
with patch.object(MultiprocDiffusionExecutor, "_init_executor"):
executor = MultiprocDiffusionExecutor(od_cfg)
executor.scheduler = scheduler
executor._closed = False
executor._processes = []
return executor
def _start_worker(req_q, res_q, count=2):
"""Simulate workers: read *count* requests from *req_q* and put
tagged ``DiffusionOutput``s on *res_q* (FIFO order).
"""
def _run():
for _ in range(count):
req = req_q.get(timeout=10)
method = req.get("method", "")
args = req.get("args", ())
if method == "generate" and args and hasattr(args[0], "request_ids"):
tag = f"result_for_{args[0].request_ids[0]}"
elif args:
tag = f"result_for_{args[0]}"
else:
tag = f"result_for_{method}"
res_q.put(_tagged_output(tag))
t = threading.Thread(target=_run, daemon=True)
t.start()
return t
def _inject_interleave(scheduler):
"""Monkey-patch ``scheduler.mq.enqueue`` so that:
* The thread named **thread_a** *blocks* after its enqueue until the
thread named **thread_b** has finished entirely.
* All other threads pass through unblocked.
Returns ``(a_enqueued: Event, b_complete: Event)`` for wiring.
"""
a_enqueued = threading.Event()
b_complete = threading.Event()
orig_enqueue = scheduler.mq.enqueue # points to req_q.put
def _controlled(item):
orig_enqueue(item)
if threading.current_thread().name == "thread_a":
a_enqueued.set() # tell B: "A has enqueued"
b_complete.wait(5) # block A until B finishes
scheduler.mq.enqueue = _controlled
return a_enqueued, b_complete
# ──────────────────── bug-reproduction: concurrent add_req ────────────────
class TestConcurrentAddReqBug:
"""Two concurrent ``Scheduler.add_req()`` calls swap results."""
def test_results_are_correctly_routed(self):
sched, req_q, res_q = _make_scheduler()
a_enqueued, b_complete = _inject_interleave(sched)
wt = _start_worker(req_q, res_q, count=2)
results: dict[str, DiffusionOutput] = {}
def _a():
results["A"] = sched.add_req(_mock_request("A"))
def _b():
a_enqueued.wait(5) # wait for A to enqueue
results["B"] = sched.add_req(_mock_request("B"))
b_complete.set() # release A
ta = threading.Thread(target=_a, name="thread_a")
tb = threading.Thread(target=_b, name="thread_b")
ta.start()
tb.start()
ta.join(10)
tb.join(10)
wt.join(5)
# With correct (locked) implementation both assertions hold.
# The bug causes them to be swapped.
assert results["A"].error == "result_for_A"
assert results["B"].error == "result_for_B"
# ──────────────── bug-reproduction: concurrent collective_rpc ─────────────
class TestConcurrentCollectiveRpcBug:
"""Two concurrent ``collective_rpc()`` calls swap results."""
def test_results_are_correctly_routed(self):
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
a_enqueued, b_complete = _inject_interleave(sched)
wt = _start_worker(req_q, res_q, count=2)
results: dict[str, object] = {}
def _a():
results["A"] = executor.collective_rpc(
"ping",
args=("call_A",),
unique_reply_rank=0,
)
def _b():
a_enqueued.wait(5)
results["B"] = executor.collective_rpc(
"ping",
args=("call_B",),
unique_reply_rank=0,
)
b_complete.set()
ta = threading.Thread(target=_a, name="thread_a")
tb = threading.Thread(target=_b, name="thread_b")
ta.start()
tb.start()
ta.join(10)
tb.join(10)
wt.join(5)
assert results["A"].error == "result_for_call_A"
assert results["B"].error == "result_for_call_B"
# ──────── bug-reproduction: add_req vs collective_rpc concurrently ────────
class TestConcurrentAddReqVsCollectiveRpcBug:
"""``add_req`` and ``collective_rpc`` running concurrently swap results."""
def test_results_are_correctly_routed(self):
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
a_enqueued, b_complete = _inject_interleave(sched)
wt = _start_worker(req_q, res_q, count=2)
results: dict[str, object] = {}
def _a(): # add_req path
results["A"] = sched.add_req(_mock_request("A"))
def _b(): # collective_rpc path
a_enqueued.wait(5)
results["B"] = executor.collective_rpc(
"ping",
args=("call_B",),
unique_reply_rank=0,
)
b_complete.set()
ta = threading.Thread(target=_a, name="thread_a")
tb = threading.Thread(target=_b, name="thread_b")
ta.start()
tb.start()
ta.join(10)
tb.join(10)
wt.join(5)
assert isinstance(results["A"], DiffusionOutput)
assert results["A"].error == "result_for_A"
assert results["B"].error == "result_for_call_B"
# ─────────────── backward-compatibility (serial) tests ────────────────────
class TestSerialOperations:
"""Verify correct behaviour for single-threaded (serial) usage.
These tests must pass both **before** and **after** any concurrency fix
is applied – they guard against regressions in the basic request path.
"""
def test_serial_add_req_returns_correct_result(self):
sched, req_q, res_q = _make_scheduler()
wt = _start_worker(req_q, res_q, count=1)
result = sched.add_req(_mock_request("X"))
wt.join(5)
assert isinstance(result, DiffusionOutput)
assert result.error == "result_for_X"
def test_serial_add_req_multiple_sequential(self):
sched, req_q, res_q = _make_scheduler()
wt = _start_worker(req_q, res_q, count=3)
for tag in ("one", "two", "three"):
out = sched.add_req(_mock_request(tag))
assert out.error == f"result_for_{tag}"
wt.join(5)
def test_serial_collective_rpc_single_rank(self):
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
wt = _start_worker(req_q, res_q, count=1)
result = executor.collective_rpc(
"ping",
args=("Y",),
unique_reply_rank=0,
)
wt.join(5)
assert result.error == "result_for_Y"
def test_serial_collective_rpc_all_ranks(self):
"""``collective_rpc`` without *unique_reply_rank* collects
``num_gpus`` responses.
"""
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
executor.od_config.num_gpus = 2
# Pre-populate two results (simulating two workers replying)
res_q.put(_tagged_output("rank0"))
res_q.put(_tagged_output("rank1"))
results = executor.collective_rpc("ping", args=("multi",))
assert len(results) == 2
assert results[0].error == "rank0"
assert results[1].error == "rank1"
def test_serial_add_req_then_collective_rpc(self):
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
wt = _start_worker(req_q, res_q, count=2)
gen_out = sched.add_req(_mock_request("gen"))
rpc_out = executor.collective_rpc(
"ping",
args=("rpc",),
unique_reply_rank=0,
)
wt.join(5)
assert gen_out.error == "result_for_gen"
assert rpc_out.error == "result_for_rpc"
def test_serial_add_req_error_propagation(self):
"""``add_req`` should raise when the worker reports an error."""
sched, _, res_q = _make_scheduler()
# Put an error response directly
res_q.put({"status": "error", "error": "boom"})
with pytest.raises(RuntimeError, match="worker error"):
sched.add_req(_mock_request("fail"))
def test_serial_collective_rpc_error_propagation(self):
"""``collective_rpc`` should raise when the worker reports an error."""
sched, _, res_q = _make_scheduler()
executor = _make_executor(sched)
res_q.put({"status": "error", "error": "kaboom"})
with pytest.raises(RuntimeError, match="kaboom"):
executor.collective_rpc("bad", unique_reply_rank=0)
def test_collective_rpc_closed_executor_raises(self):
sched, _, _ = _make_scheduler()
executor = _make_executor(sched)
executor._closed = True
with pytest.raises(RuntimeError, match="closed"):
executor.collective_rpc("anything")
# ─────────── timeout regression: RPC must not block on a stalled lock ─────
class TestCollectiveRpcTimeoutWhileLockHeld:
"""``collective_rpc(timeout=...)`` must honour its timeout even when
another thread holds ``scheduler._lock`` indefinitely (e.g. a stalled
``add_req`` waiting on an unresponsive worker).
"""
def test_rpc_times_out_when_lock_held_directly(self):
"""Simplest case: lock is manually held by another thread."""
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
stall_started = threading.Event()
def _hold_lock():
sched._lock.acquire()
stall_started.set()
# Hold the lock far longer than the RPC timeout.
threading.Event().wait(30)
sched._lock.release()
stall_thread = threading.Thread(target=_hold_lock, daemon=True)
stall_thread.start()
stall_started.wait(5)
# collective_rpc should raise TimeoutError, NOT block forever.
with pytest.raises(TimeoutError):
executor.collective_rpc("health", timeout=0.5)
def test_rpc_times_out_when_add_req_stalled_on_worker(self):
"""Real-world scenario the bot flagged:
``add_req`` holds ``_lock`` while blocked on ``result_mq.dequeue()``
because the worker never replies. A concurrent
``collective_rpc(timeout=...)`` must still time out instead of
hanging forever waiting for the lock.
"""
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
add_req_blocked = threading.Event()
# Patch dequeue: signal once entered, then block indefinitely
# (simulates a worker that never sends a result).
orig_dequeue = sched.result_mq.dequeue
def _hanging_dequeue(timeout=None):
add_req_blocked.set()
# Block forever — the worker is "hung".
threading.Event().wait(30)
return orig_dequeue(timeout=timeout)
sched.result_mq.dequeue = _hanging_dequeue
# Thread running add_req — acquires the lock, enqueues, then
# blocks on dequeue forever (worker hang).
def _stalled_add_req():
try:
sched.add_req(_mock_request("stalled"))
except Exception:
pass
t = threading.Thread(target=_stalled_add_req, daemon=True)
t.start()
# Wait until add_req is truly inside the lock and blocking.
add_req_blocked.wait(5)
# collective_rpc should time out at lock acquisition, not hang.
with pytest.raises(TimeoutError):
executor.collective_rpc("health_check", timeout=0.5)
def test_rpc_without_timeout_still_waits_for_lock(self):
"""When no timeout is given, ``collective_rpc`` should still wait
for the lock (blocking) — existing behaviour preserved.
"""
sched, req_q, res_q = _make_scheduler()
executor = _make_executor(sched)
lock_released = threading.Event()
def _hold_and_release():
sched._lock.acquire()
# Hold for a short time then release.
threading.Event().wait(0.3)
sched._lock.release()
lock_released.set()
# Pre-populate a result so collective_rpc succeeds after lock.
res_q.put(_tagged_output("ok"))
t = threading.Thread(target=_hold_and_release, daemon=True)
t.start()
# No timeout → should block until lock is released, then succeed.
result = executor.collective_rpc(
"ping",
args=("wait",),
unique_reply_rank=0,
)
t.join(5)
assert result.error == "ok"
+170 -165
View File
@@ -13,9 +13,9 @@ This module tests the WorkerWrapperBase implementation:
"""
from typing import Any
from unittest.mock import Mock, patch
import pytest
from pytest_mock import MockerFixture
from vllm_omni.diffusion.worker.diffusion_worker import (
CustomPipelineWorkerExtension,
@@ -23,15 +23,18 @@ from vllm_omni.diffusion.worker.diffusion_worker import (
WorkerWrapperBase,
)
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
# -------------------------------------------------------------------------
# Fixtures
# -------------------------------------------------------------------------
@pytest.fixture
def mock_od_config():
def mock_od_config(mocker: MockerFixture):
"""Create a mock OmniDiffusionConfig for use in tests."""
config = Mock()
config = mocker.Mock()
config.num_gpus = 1
config.master_port = 12345
config.enable_sleep_mode = False
@@ -72,9 +75,10 @@ class MockCustomPipeline:
class TestWorkerWrapperBaseInitialization:
"""Test WorkerWrapperBase initialization behavior."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_basic_initialization(self, mock_worker_init, mock_od_config):
def test_basic_initialization(self, mocker: MockerFixture, mock_od_config):
"""Test basic initialization without extensions."""
mock_worker_init = mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
@@ -103,49 +107,49 @@ class TestWorkerWrapperBaseInitialization:
class TestWorkerWrapperBaseExtension:
"""Test WorkerWrapperBase worker extension functionality."""
def test_prepare_worker_class_without_extension(self, mock_od_config):
def test_prepare_worker_class_without_extension(self, mocker: MockerFixture, mock_od_config):
"""Test _prepare_worker_class without a worker extension."""
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
)
worker_class = wrapper._prepare_worker_class()
assert worker_class == DiffusionWorker
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
)
worker_class = wrapper._prepare_worker_class()
assert worker_class == DiffusionWorker
def test_prepare_worker_class_with_extension_class(self, mock_od_config):
def test_prepare_worker_class_with_extension_class(self, mocker: MockerFixture, mock_od_config):
"""Test _prepare_worker_class with an explicit extension class."""
class TestExtension:
def custom_method(self):
return "extension_method"
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
assert hasattr(wrapper.worker.__class__, "custom_method")
assert TestExtension in wrapper.worker.__class__.__bases__
assert hasattr(wrapper.worker.__class__, "custom_method")
assert TestExtension in wrapper.worker.__class__.__bases__
@patch("vllm.utils.import_utils.resolve_obj_by_qualname")
def test_prepare_worker_class_with_extension_string(self, mock_resolve, mock_od_config):
def test_prepare_worker_class_with_extension_string(self, mocker: MockerFixture, mock_od_config):
"""Test _prepare_worker_class with worker extension as string."""
mock_resolve = mocker.patch("vllm.utils.import_utils.resolve_obj_by_qualname")
mock_resolve.return_value = TestExtension
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls="tests.diffusion.test_worker_wrapper_base.TestExtension",
)
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls="tests.diffusion.test_worker_wrapper_base.TestExtension",
)
assert hasattr(wrapper.worker.__class__, "custom_method")
assert hasattr(wrapper.worker.__class__, "custom_method")
# -------------------------------------------------------------------------
@@ -156,70 +160,70 @@ class TestWorkerWrapperBaseExtension:
class TestWorkerWrapperBaseDelegation:
"""Test WorkerWrapperBase delegation to wrapped worker."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_generate_delegation(self, mock_worker_init, mock_od_config):
def test_generate_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that generate() delegates to worker.generate()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
mock_output = Mock()
wrapper.worker.generate = Mock(return_value=mock_output)
mock_output = mocker.Mock()
wrapper.worker.generate = mocker.Mock(return_value=mock_output)
mock_requests = [Mock()]
mock_requests = [mocker.Mock()]
result = wrapper.generate(mock_requests)
wrapper.worker.generate.assert_called_once_with(mock_requests)
assert result == mock_output
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_model_delegation(self, mock_worker_init, mock_od_config):
def test_execute_model_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that execute_model() delegates to worker.execute_model()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
mock_output = Mock()
wrapper.worker.execute_model = Mock(return_value=mock_output)
mock_output = mocker.Mock()
wrapper.worker.execute_model = mocker.Mock(return_value=mock_output)
mock_reqs = [Mock()]
mock_reqs = [mocker.Mock()]
result = wrapper.execute_model(mock_reqs, mock_od_config)
wrapper.worker.execute_model.assert_called_once_with(mock_reqs, mock_od_config)
assert result == mock_output
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_load_weights_delegation(self, mock_worker_init, mock_od_config):
def test_load_weights_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that load_weights() delegates to worker.load_weights()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
expected_result = {"weight1", "weight2"}
wrapper.worker.load_weights = Mock(return_value=expected_result)
wrapper.worker.load_weights = mocker.Mock(return_value=expected_result)
mock_weights = [("weight1", Mock()), ("weight2", Mock())]
mock_weights = [("weight1", mocker.Mock()), ("weight2", mocker.Mock())]
result = wrapper.load_weights(mock_weights)
wrapper.worker.load_weights.assert_called_once_with(mock_weights)
assert result == expected_result
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_sleep_delegation(self, mock_worker_init, mock_od_config):
def test_sleep_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that sleep() delegates to worker.sleep()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.sleep = Mock(return_value=True)
wrapper.worker.sleep = mocker.Mock(return_value=True)
result = wrapper.sleep(level=1)
wrapper.worker.sleep.assert_called_once_with(1)
assert result is True
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_wake_up_delegation(self, mock_worker_init, mock_od_config):
def test_wake_up_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that wake_up() delegates to worker.wake_up()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.wake_up = Mock(return_value=True)
wrapper.worker.wake_up = mocker.Mock(return_value=True)
result = wrapper.wake_up(tags=["weights"])
wrapper.worker.wake_up.assert_called_once_with(["weights"])
assert result is True
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_shutdown_delegation(self, mock_worker_init, mock_od_config):
def test_shutdown_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test that shutdown() delegates to worker.shutdown()."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.shutdown = Mock(return_value=None)
wrapper.worker.shutdown = mocker.Mock(return_value=None)
result = wrapper.shutdown()
wrapper.worker.shutdown.assert_called_once()
@@ -234,39 +238,39 @@ class TestWorkerWrapperBaseDelegation:
class TestWorkerWrapperBaseExecuteMethod:
"""Test WorkerWrapperBase.execute_method functionality."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_success(self, mock_worker_init, mock_od_config):
def test_execute_method_success(self, mocker: MockerFixture, mock_od_config):
"""Test execute_method successfully calls worker method."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.test_method = Mock(return_value="method_result")
wrapper.worker.test_method = mocker.Mock(return_value="method_result")
result = wrapper.execute_method("test_method", "arg1", kwarg1="value1")
wrapper.worker.test_method.assert_called_once_with("arg1", kwarg1="value1")
assert result == "method_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_with_no_args(self, mock_worker_init, mock_od_config):
def test_execute_method_with_no_args(self, mocker: MockerFixture, mock_od_config):
"""Test execute_method with no arguments."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.no_args_method = Mock(return_value="no_args_result")
wrapper.worker.no_args_method = mocker.Mock(return_value="no_args_result")
result = wrapper.execute_method("no_args_method")
wrapper.worker.no_args_method.assert_called_once_with()
assert result == "no_args_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_error(self, mock_worker_init, mock_od_config):
def test_execute_method_error(self, mocker: MockerFixture, mock_od_config):
"""Test execute_method raises exception on error."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.error_method = Mock(side_effect=RuntimeError("Test error"))
wrapper.worker.error_method = mocker.Mock(side_effect=RuntimeError("Test error"))
with pytest.raises(RuntimeError, match="Test error"):
wrapper.execute_method("error_method")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_execute_method_invalid_type(self, mock_worker_init, mock_od_config):
def test_execute_method_invalid_type(self, mocker: MockerFixture, mock_od_config):
"""Test execute_method with invalid method type."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
with pytest.raises(AssertionError, match="Method must be str"):
@@ -281,26 +285,26 @@ class TestWorkerWrapperBaseExecuteMethod:
class TestWorkerWrapperBaseGetAttr:
"""Test WorkerWrapperBase.__getattr__ delegation."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_delegation(self, mock_worker_init, mock_od_config):
def test_getattr_delegation(self, mocker: MockerFixture, mock_od_config):
"""Test __getattr__ delegates to worker attributes."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.custom_attribute = "test_value"
assert wrapper.custom_attribute == "test_value"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_method_access(self, mock_worker_init, mock_od_config):
def test_getattr_method_access(self, mocker: MockerFixture, mock_od_config):
"""Test __getattr__ delegates to worker methods."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
wrapper.worker.custom_method = Mock(return_value="method_result")
wrapper.worker.custom_method = mocker.Mock(return_value="method_result")
result = wrapper.custom_method()
wrapper.worker.custom_method.assert_called_once()
assert result == "method_result"
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_getattr_missing_attribute(self, mock_worker_init, mock_od_config):
def test_getattr_missing_attribute(self, mocker: MockerFixture, mock_od_config):
"""Test __getattr__ raises AttributeError for missing attributes."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(gpu_id=0, od_config=mock_od_config, base_worker_class=DiffusionWorker)
with pytest.raises(AttributeError):
_ = wrapper.nonexistent_attribute
@@ -314,47 +318,46 @@ class TestWorkerWrapperBaseGetAttr:
class TestWorkerWrapperBaseEdgeCases:
"""Test WorkerWrapperBase edge cases and special scenarios."""
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_extension_conflict_warning(self, mock_worker_init, mock_od_config, caplog):
def test_extension_conflict_warning(self, mocker: MockerFixture, mock_od_config, caplog):
"""Test a warning is logged when an extension conflicts with worker."""
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
class ConflictExtension:
def load_model(self):
return "extension_load_model"
with patch.object(DiffusionWorker, "load_model"):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=ConflictExtension,
)
assert wrapper.worker is not None
mocker.patch.object(DiffusionWorker, "load_model")
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=ConflictExtension,
)
assert wrapper.worker is not None
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_multiple_extensions_same_class(self, mock_worker_init, mock_od_config):
def test_multiple_extensions_same_class(self, mocker: MockerFixture, mock_od_config):
"""Test that applying same extension twice doesn't duplicate it."""
class TestExtension:
def custom_method(self):
return "extension"
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper1 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
wrapper2 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper1 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
wrapper2 = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=TestExtension,
)
assert hasattr(wrapper1.worker, "custom_method")
assert hasattr(wrapper2.worker, "custom_method")
assert hasattr(wrapper1.worker, "custom_method")
assert hasattr(wrapper2.worker, "custom_method")
# -------------------------------------------------------------------------
@@ -365,11 +368,12 @@ class TestWorkerWrapperBaseEdgeCases:
class TestCustomPipelineWorkerExtension:
"""Test CustomPipelineWorkerExtension functionality."""
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_basic(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
def test_re_init_pipeline_basic(self, mocker: MockerFixture, mock_od_config):
"""Test basic re_init_pipeline functionality."""
mocker.patch("torch.cuda.empty_cache")
mocker.patch("gc.collect")
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
@@ -378,12 +382,12 @@ class TestCustomPipelineWorkerExtension:
)
# Setup mock model_runner and pipeline
mock_model_runner = Mock()
mock_pipeline = Mock()
mock_model_runner = mocker.Mock()
mock_pipeline = mocker.Mock()
mock_model_runner.pipeline = mock_pipeline
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
wrapper.worker.init_lora_manager = mocker.Mock()
wrapper.worker.load_model = mocker.Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
@@ -397,11 +401,12 @@ class TestCustomPipelineWorkerExtension:
)
wrapper.worker.init_lora_manager.assert_called_once()
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_cleanup(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
def test_re_init_pipeline_cleanup(self, mocker: MockerFixture, mock_od_config):
"""Test that re_init_pipeline properly cleans up old pipeline."""
mock_gc_collect = mocker.patch("gc.collect")
mock_empty_cache = mocker.patch("torch.cuda.empty_cache")
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
@@ -410,12 +415,12 @@ class TestCustomPipelineWorkerExtension:
)
# Setup mock model_runner with pipeline
mock_model_runner = Mock()
mock_pipeline = Mock()
mock_model_runner = mocker.Mock()
mock_pipeline = mocker.Mock()
mock_model_runner.pipeline = mock_pipeline
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
wrapper.worker.init_lora_manager = mocker.Mock()
wrapper.worker.load_model = mocker.Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
@@ -426,11 +431,12 @@ class TestCustomPipelineWorkerExtension:
mock_gc_collect.assert_called_once()
mock_empty_cache.assert_called_once()
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_none_pipeline(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
def test_re_init_pipeline_none_pipeline(self, mocker: MockerFixture, mock_od_config):
"""Test re_init_pipeline when pipeline is None."""
mocker.patch("torch.cuda.empty_cache")
mocker.patch("gc.collect")
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
@@ -439,11 +445,11 @@ class TestCustomPipelineWorkerExtension:
)
# Setup mock model_runner with None pipeline
mock_model_runner = Mock()
mock_model_runner = mocker.Mock()
mock_model_runner.pipeline = None
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
wrapper.worker.init_lora_manager = mocker.Mock()
wrapper.worker.load_model = mocker.Mock()
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
@@ -453,32 +459,30 @@ class TestCustomPipelineWorkerExtension:
# Verify load_model was still called
wrapper.worker.load_model.assert_called_once()
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_custom_pipeline_args_initialization(self, mock_worker_init, mock_od_config):
def test_custom_pipeline_args_initialization(self, mocker: MockerFixture, mock_od_config):
"""Test initialization with custom_pipeline_args calls re_init_pipeline."""
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
with patch.object(DiffusionWorker, "__init__", return_value=None):
with patch.object(WorkerWrapperBase, "_prepare_worker_class") as mock_prepare:
# Create a mock worker class with re_init_pipeline
mock_worker_class = Mock()
mock_worker_instance = Mock()
mock_worker_instance.re_init_pipeline = Mock()
mock_worker_class.return_value = mock_worker_instance
mock_prepare.return_value = mock_worker_class
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
mock_prepare = mocker.patch.object(WorkerWrapperBase, "_prepare_worker_class")
# Create a mock worker class with re_init_pipeline
mock_worker_class = mocker.Mock()
mock_worker_instance = mocker.Mock()
mock_worker_instance.re_init_pipeline = mocker.Mock()
mock_worker_class.return_value = mock_worker_instance
mock_prepare.return_value = mock_worker_class
_ = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
custom_pipeline_args=custom_args,
)
_ = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
custom_pipeline_args=custom_args,
)
# Verify re_init_pipeline was called with custom_pipeline_args
mock_worker_instance.re_init_pipeline.assert_called_once_with(custom_args)
# Verify re_init_pipeline was called with custom_pipeline_args
mock_worker_instance.re_init_pipeline.assert_called_once_with(custom_args)
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_custom_pipeline_with_explicit_extension(self, mock_worker_init, mock_od_config):
def test_custom_pipeline_with_explicit_extension(self, mocker: MockerFixture, mock_od_config):
"""Test that explicit worker_extension_cls is preserved when custom_pipeline_args is provided."""
class CustomExtension:
@@ -490,24 +494,25 @@ class TestCustomPipelineWorkerExtension:
custom_args = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
with patch.object(DiffusionWorker, "__init__", return_value=None):
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomExtension,
custom_pipeline_args=custom_args,
)
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
base_worker_class=DiffusionWorker,
worker_extension_cls=CustomExtension,
custom_pipeline_args=custom_args,
)
# Should still have the explicitly provided extension
assert CustomExtension in wrapper.worker.__class__.__bases__
assert hasattr(wrapper.worker, "custom_extension_method")
# Should still have the explicitly provided extension
assert CustomExtension in wrapper.worker.__class__.__bases__
assert hasattr(wrapper.worker, "custom_extension_method")
@patch("torch.cuda.empty_cache")
@patch("gc.collect")
@patch.object(DiffusionWorker, "__init__", return_value=None)
def test_re_init_pipeline_multiple_calls(self, mock_worker_init, mock_gc_collect, mock_empty_cache, mock_od_config):
def test_re_init_pipeline_multiple_calls(self, mocker: MockerFixture, mock_od_config):
"""Test calling re_init_pipeline multiple times."""
mocker.patch("torch.cuda.empty_cache")
mocker.patch("gc.collect")
mocker.patch.object(DiffusionWorker, "__init__", return_value=None)
wrapper = WorkerWrapperBase(
gpu_id=0,
od_config=mock_od_config,
@@ -516,13 +521,13 @@ class TestCustomPipelineWorkerExtension:
)
# Setup mock model_runner
mock_model_runner = Mock()
mock_pipeline1 = Mock()
mock_pipeline2 = Mock()
mock_model_runner = mocker.Mock()
mock_pipeline1 = mocker.Mock()
mock_pipeline2 = mocker.Mock()
mock_model_runner.pipeline = mock_pipeline1
wrapper.worker.model_runner = mock_model_runner
wrapper.worker.init_lora_manager = Mock()
wrapper.worker.load_model = Mock()
wrapper.worker.init_lora_manager = mocker.Mock()
wrapper.worker.load_model = mocker.Mock()
# First call
custom_args1 = {"pipeline_class": "tests.diffusion.test_worker_wrapper_base.MockCustomPipeline"}
@@ -1,19 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def mock_objects():
return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
def mock_objects(mocker: MockerFixture):
return {"connector": mocker.MagicMock(), "metrics": mocker.MagicMock(), "queue_fn": mocker.MagicMock()}
def test_send_success(mock_objects):
@@ -138,7 +139,7 @@ def test_recv_no_connector():
assert inputs is None
def test_shm_connector_flow():
def test_shm_connector_flow(mocker: MockerFixture):
"""
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
Using real SharedMemoryConnector (inline mode for simplicity).
@@ -161,7 +162,7 @@ def test_shm_connector_flow():
def mock_submit(payload):
queue_capture.append(payload)
mock_metrics = MagicMock()
mock_metrics = mocker.MagicMock()
# 3. Send
success = try_send_via_connector(
@@ -217,7 +218,7 @@ def test_get_connectors_for_stage():
assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
def test_recv_with_missing_metadata():
def test_recv_with_missing_metadata(mocker: MockerFixture):
"""Test recv when queue payload is malformed (missing metadata)."""
# Connector expects metadata but task doesn't have it
task = {
@@ -226,7 +227,7 @@ def test_recv_with_missing_metadata():
"from_stage": "0",
# Missing "connector_metadata"
}
mock_conn = MagicMock()
mock_conn = mocker.MagicMock()
# If get is called with None metadata, connector usually handles it or adapter handles exception
mock_conn.get.side_effect = Exception("Get failed")
@@ -1,14 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
# pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_basic_serialization():
"""Test basic msgpack serialization."""
@@ -78,19 +80,19 @@ def test_put_get_inline(shm_connector):
assert size == ret_size
def test_put_get_shm(shm_connector, monkeypatch):
def test_put_get_shm(mocker: MockerFixture, shm_connector, monkeypatch: pytest.MonkeyPatch):
"""Test SHM transfer logic for large data (Mocked)."""
# Create data larger than 100 bytes
data = {"large": "x" * 200}
# Mock SHM return values
mock_handle = {"name": "test_shm", "size": 200}
mock_write = MagicMock(return_value=mock_handle)
mock_write = mocker.MagicMock(return_value=mock_handle)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
# When reading, return the serialized bytes of the data
serialized_data = shm_connector.serialize_obj(data)
mock_read = MagicMock(return_value=serialized_data)
mock_read = mocker.MagicMock(return_value=serialized_data)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
# Put
@@ -4,10 +4,10 @@
import threading
from collections import deque
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from pytest_mock import MockerFixture
from vllm.v1.request import RequestStatus
from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
@@ -35,25 +35,25 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
prompt_token_ids=[],
num_computed_tokens=0,
additional_information=None,
is_finished=lambda: status == RequestStatus.FINISHED_STOPPED,
)
@pytest.fixture
def build_adapter(monkeypatch):
def build_adapter(monkeypatch, mocker: MockerFixture):
def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2):
connector = MagicMock()
connector = mocker.MagicMock()
connector.stage_id = stage_id
connector.get.return_value = None
connector.put.return_value = (True, 1, {})
def _fake_base_init(self, config):
self.config = config
self._pending_load_reqs = {}
self._pending_load_reqs = deque()
self._finished_load_reqs = set()
self._pending_save_reqs = {}
self._pending_save_reqs = deque()
self._finished_save_reqs = set()
self.stop_event = threading.Event()
self.lock = threading.Lock()
monkeypatch.setattr(OmniTransferAdapterBase, "__init__", _fake_base_init)
monkeypatch.setattr(
@@ -108,9 +108,8 @@ def test_load_poll(build_adapter):
adapter.load_async(request)
payload = {"code_predictor_codes": [[1]], "hidden_states": torch.tensor([[2.0]]), "finished": True}
connector.get.return_value = (payload, 16)
adapter._poll_single_request("req-1")
adapter._poll_single_request(request)
connector.get.assert_called_once_with("1", "2", "external-1_1_0")
assert request.additional_information == payload
assert adapter.get_req_chunk["req-1"] == 1
assert "req-1" in adapter._finished_load_reqs
@@ -120,17 +119,15 @@ def test_load_poll(build_adapter):
def test_save_async(build_adapter):
adapter, _ = build_adapter(stage_id=1)
request = SimpleNamespace(external_req_id="external-1")
request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1")
adapter.custom_process_next_stage_input_func = lambda **kwargs: {"x": [1], "finished": False}
adapter.save_async(pooling_output=None, request=request)
adapter.custom_process_next_stage_input_func = lambda **kwargs: {}
adapter.save_async(pooling_output=None, request=request)
assert adapter.put_req_chunk["external-1"] == 1
queued = adapter._pending_save_reqs["external-1"]
assert len(queued) == 1
assert queued[0]["put_key"] == "external-1_1_0"
task = adapter._pending_save_reqs.popleft()
assert task["is_finished"] is False
def test_update_request_payload(build_adapter):
@@ -180,3 +177,232 @@ def test_postprocess_scheduler_output(build_adapter):
assert cached_info["cached-ready"] == {"k": "v"}
assert cached_info["missing"] is None
assert adapter.requests_with_ready_chunks == {"leftover"}
# ---------------------------------------------------------------
# Cleanup tests
# ---------------------------------------------------------------
def _populate_adapter_state(adapter, req_id="req-1", ext_id="ext-1"):
"""Fill every per-request structure so cleanup can be verified."""
adapter.finished_requests.add(req_id)
adapter.get_req_chunk[req_id] = 3
adapter.requests_with_ready_chunks.add(req_id)
adapter.request_ids_mapping[req_id] = ext_id
adapter._pending_load_reqs.append(SimpleNamespace(request_id=req_id))
adapter._finished_load_reqs.add(req_id)
adapter.put_req_chunk[ext_id] = 5
adapter.request_payload[ext_id] = {"hidden": [1, 2]}
adapter.code_prompt_token_ids[ext_id] = [[10, 20]]
def test_cleanup_clears_all_state(build_adapter):
"""After cleanup, no per-request key should remain in any dict/set."""
adapter, _ = build_adapter(stage_id=1)
req_id, ext_id = "req-1", "ext-1"
_populate_adapter_state(adapter, req_id, ext_id)
adapter.cleanup(req_id, ext_id)
assert req_id not in adapter.finished_requests
assert req_id not in adapter.get_req_chunk
assert req_id not in adapter.requests_with_ready_chunks
assert req_id not in adapter.request_ids_mapping
assert all(getattr(r, "request_id", None) != req_id for r in adapter._pending_load_reqs)
assert req_id not in adapter._finished_load_reqs
assert ext_id not in adapter.put_req_chunk
assert ext_id not in adapter.request_payload
assert ext_id not in adapter.code_prompt_token_ids
def test_cleanup_infers_external_id(build_adapter):
"""When external_req_id is None, cleanup should look it up from the mapping."""
adapter, _ = build_adapter(stage_id=1)
req_id, ext_id = "req-2", "ext-2"
_populate_adapter_state(adapter, req_id, ext_id)
adapter.cleanup(req_id)
assert ext_id not in adapter.put_req_chunk
assert ext_id not in adapter.request_payload
def test_cleanup_idempotent(build_adapter):
"""Calling cleanup multiple times for the same (or nonexistent) request must not raise."""
adapter, _ = build_adapter(stage_id=1)
try:
adapter.cleanup("nonexistent")
adapter.cleanup("nonexistent")
except Exception as e:
pytest.fail(f"cleanup should be idempotent: {e}")
req_id, ext_id = "req-3", "ext-3"
_populate_adapter_state(adapter, req_id, ext_id)
adapter.cleanup(req_id, ext_id)
try:
adapter.cleanup(req_id, ext_id)
except Exception as e:
pytest.fail(f"second cleanup should be idempotent: {e}")
def test_cleanup_request_id_reuse_not_polluted(build_adapter):
"""After cleanup, reusing the same request_id must not be treated as finished."""
adapter, _ = build_adapter(stage_id=1)
req_id, ext_id = "req-reuse", "ext-reuse"
_populate_adapter_state(adapter, req_id, ext_id)
adapter.cleanup(req_id, ext_id)
assert req_id not in adapter.finished_requests
assert req_id not in adapter.get_req_chunk
def test_cleanup_preserves_pending_save(build_adapter):
"""Cleanup must NOT remove _pending_save_reqs to avoid losing unsent chunks."""
adapter, _ = build_adapter(stage_id=1)
req_id, ext_id = "req-4", "ext-4"
_populate_adapter_state(adapter, req_id, ext_id)
pending_task = {"put_key": f"{ext_id}_1_0", "data": {"x": 1}}
adapter._pending_save_reqs.append(pending_task)
adapter.cleanup(req_id, ext_id)
assert len(adapter._pending_save_reqs) == 1
def test_cleanup_only_affects_target_request(build_adapter):
"""Cleanup for one request must not affect another request's state."""
adapter, _ = build_adapter(stage_id=1)
_populate_adapter_state(adapter, "req-a", "ext-a")
_populate_adapter_state(adapter, "req-b", "ext-b")
adapter.cleanup("req-a", "ext-a")
assert "req-b" in adapter.finished_requests
assert "req-b" in adapter.get_req_chunk
assert "ext-b" in adapter.put_req_chunk
assert "ext-b" in adapter.request_payload
assert "ext-b" in adapter.code_prompt_token_ids
assert "req-b" in adapter.request_ids_mapping
def test_cleanup_after_poll_flow(build_adapter):
"""Simulate full load_async -> poll -> finished -> cleanup cycle."""
adapter, connector = build_adapter(stage_id=2, model_mode="ar")
request = _req("req-flow", RequestStatus.WAITING, external_req_id="ext-flow")
adapter.load_async(request)
adapter.request_ids_mapping["req-flow"] = "ext-flow"
payload = {"hidden_states": torch.tensor([[1.0]]), "finished": True}
connector.get.return_value = (payload, 8)
adapter._poll_single_request(request)
assert "req-flow" in adapter.finished_requests
assert adapter.get_req_chunk["req-flow"] == 1
assert "req-flow" in adapter.request_ids_mapping
adapter.cleanup("req-flow", "ext-flow")
assert "req-flow" not in adapter.finished_requests
assert "req-flow" not in adapter.get_req_chunk
assert "req-flow" not in adapter.request_ids_mapping
assert "ext-flow" not in adapter.request_payload
# ---------------------------------------------------------------
# Scheduler trigger tests
# ---------------------------------------------------------------
class _HashableRequest(SimpleNamespace):
"""SimpleNamespace that can be added to a set (needed by scheduler internals)."""
def __hash__(self):
return hash(self.request_id)
def __eq__(self, other):
return getattr(other, "request_id", None) == self.request_id
def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: MockerFixture):
"""OmniGenerationScheduler must call adapter.cleanup when request finishes."""
cleanup_calls = []
adapter_mock = mocker.MagicMock()
adapter_mock.finished_requests = {"req-s1"}
adapter_mock.cleanup = lambda *a, **kw: cleanup_calls.append((a, kw))
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
scheduler = mocker.MagicMock()
scheduler.chunk_transfer_adapter = adapter_mock
scheduler.connector = None
scheduler.ec_connector = None
scheduler.perf_metrics = None
scheduler.log_stats = False
scheduler.recompute_kv_load_failures = False
scheduler.structured_output_manager = mocker.MagicMock()
scheduler.structured_output_manager.should_advance.return_value = False
scheduler.finished_req_ids_dict = {}
scheduler.kv_cache_manager.take_events.return_value = None
scheduler.kv_event_publisher = mocker.MagicMock()
request = _HashableRequest(
request_id="req-s1",
external_req_id="ext-s1",
status=RequestStatus.RUNNING,
is_finished=lambda: False,
num_computed_tokens=10,
num_prompt_tokens=10,
prompt_token_ids=list(range(10)),
num_output_placeholders=0,
sampling_params=None,
pooling_params=None,
stop_reason=None,
client_index=0,
take_events=lambda: [],
trace_headers=None,
num_cached_tokens=0,
num_external_computed_tokens=0,
num_nans_in_logits=0,
get_finished_reason=lambda: "stop",
)
scheduler.requests = {"req-s1": request}
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
scheduler._free_request = mocker.MagicMock(return_value=None)
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
scheduler.running = [request]
scheduler.waiting = mocker.MagicMock()
scheduler.waiting.remove_requests = mocker.MagicMock()
scheduler.make_stats = mocker.MagicMock(return_value=None)
scheduler_output = SimpleNamespace(
num_scheduled_tokens={"req-s1": 10},
scheduled_spec_decode_tokens={},
num_invalid_spec_tokens=0,
)
model_runner_output = SimpleNamespace(
sampled_token_ids=None,
logprobs=None,
prompt_logprobs_dict={},
pooler_output=None,
num_nans_in_logits=None,
kv_connector_output=None,
cudagraph_stats=None,
req_id_to_index={"req-s1": 0},
)
OmniGenerationScheduler.update_from_output(scheduler, scheduler_output, model_runner_output)
assert len(cleanup_calls) == 1
args, _ = cleanup_calls[0]
assert args[0] == "req-s1"
assert args[1] == "ext-s1"
@@ -7,7 +7,6 @@ These tests do NOT require Mooncake or RDMA environment.
"""
import threading
import unittest
import pytest
import torch
@@ -22,7 +21,7 @@ pytestmark = [pytest.mark.cpu, pytest.mark.parallel]
@pytest.mark.core_model
class TestBufferAllocator(unittest.TestCase):
class TestBufferAllocator:
"""Unit tests for BufferAllocator."""
def test_basic_alloc_free(self):
@@ -30,15 +29,15 @@ class TestBufferAllocator(unittest.TestCase):
allocator = BufferAllocator(total_size=4096, alignment=64)
offset1 = allocator.alloc(512)
self.assertEqual(offset1, 0)
assert offset1 == 0
offset2 = allocator.alloc(512)
self.assertGreater(offset2, 0)
assert offset2 > 0
# Free first block, should be reusable
allocator.free(offset1, 512)
offset3 = allocator.alloc(512)
self.assertEqual(offset3, 0)
assert offset3 == 0
def test_alignment(self):
"""Verify allocation respects alignment."""
@@ -47,22 +46,22 @@ class TestBufferAllocator(unittest.TestCase):
_offset1 = allocator.alloc(100)
offset2 = allocator.alloc(100)
self.assertEqual(offset2 % 128, 0)
self.assertEqual(offset2, 128)
assert offset2 % 128 == 0
assert offset2 == 128
def test_exhaustion_and_recovery(self):
"""Test that full allocation fails, then succeeds after free."""
allocator = BufferAllocator(total_size=1024, alignment=64)
offset = allocator.alloc(1024)
self.assertEqual(offset, 0)
assert offset == 0
with self.assertRaises(MemoryError):
with pytest.raises(MemoryError):
allocator.alloc(64)
allocator.free(offset, 1024)
offset2 = allocator.alloc(1024)
self.assertEqual(offset2, 0)
assert offset2 == 0
def test_thread_safety(self):
"""Verify allocator is thread-safe under concurrent access."""
@@ -84,10 +83,10 @@ class TestBufferAllocator(unittest.TestCase):
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"Thread safety errors: {errors}")
assert len(errors) == 0, f"Thread safety errors: {errors}"
class TestAllocatorInvariants(unittest.TestCase):
class TestAllocatorInvariants:
"""
Defensive invariant tests for BufferAllocator: double-free, partial
overlap corruption, and adjacent-block merging.
@@ -106,7 +105,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(offset, 256)
# Pool should still be consistent: allocate full size back
offset2 = allocator.alloc(4096)
self.assertEqual(offset2, 0)
assert offset2 == 0
@pytest.mark.slow
def test_double_free_after_merge_is_safe(self):
@@ -123,7 +122,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(a, 64) # should not raise
# Pool should still be fully usable
offset = allocator.alloc(4096)
self.assertEqual(offset, 0)
assert offset == 0
@pytest.mark.slow
def test_partial_overlap_raises_corruption(self):
@@ -133,7 +132,7 @@ class TestAllocatorInvariants(unittest.TestCase):
b = allocator.alloc(128)
allocator.free(a, 128) # [0, 128) is now free
# Try to free a region that starts inside [0,128) but extends beyond
with self.assertRaises(RuntimeError):
with pytest.raises(RuntimeError):
allocator.free(64, 128) # [64, 192) overlaps with free [0, 128)
# b is still allocated; freeing b should be fine
@@ -154,7 +153,7 @@ class TestAllocatorInvariants(unittest.TestCase):
# covering at least 192 bytes (3 * 64).
# Verify by allocating a contiguous block of 192 bytes.
offset = allocator.alloc(192)
self.assertEqual(offset, 0, "Adjacent blocks were not merged properly")
assert offset == 0, "Adjacent blocks were not merged properly"
@pytest.mark.slow
def test_fragmentation_and_defrag(self):
@@ -174,7 +173,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(d, 64) # free blocks: [64, 128) and [192, 256)
# Pool has two 64-byte holes; contiguous 128 is unavailable
with self.assertRaises(MemoryError):
with pytest.raises(MemoryError):
allocator.alloc(128)
allocator.free(a, 64)
@@ -182,14 +181,16 @@ class TestAllocatorInvariants(unittest.TestCase):
# After freeing everything, full pool should be available
offset = allocator.alloc(256)
self.assertEqual(offset, 0)
assert offset == 0
@pytest.mark.core_model
class TestManagedBuffer(unittest.TestCase):
class TestManagedBuffer:
"""Unit tests for ManagedBuffer."""
def setUp(self):
@pytest.fixture(autouse=True)
def _setup(self):
# automatically invoked for every test method in the class
self.allocator = BufferAllocator(total_size=4096, alignment=64)
self.pool = torch.zeros(4096, dtype=torch.uint8)
@@ -203,12 +204,12 @@ class TestManagedBuffer(unittest.TestCase):
self.pool[offset : offset + 64] = src.view(torch.uint8)
# Raw uint8 view
self.assertEqual(buf.tensor.shape[0], 64)
assert buf.tensor.shape[0] == 64
# Typed view
typed = buf.as_tensor(dtype=torch.float32, shape=(4, 4))
self.assertEqual(typed.shape, (4, 4))
self.assertTrue(torch.equal(typed.flatten(), src))
assert typed.shape == (4, 4)
assert torch.equal(typed.flatten(), src)
buf.release()
@@ -217,14 +218,10 @@ class TestManagedBuffer(unittest.TestCase):
offset = self.allocator.alloc(128)
with ManagedBuffer(self.allocator, offset, 128, self.pool) as buf:
self.assertFalse(buf._released)
assert not buf._released
self.assertTrue(buf._released)
assert buf._released
# Space should be reusable
new_offset = self.allocator.alloc(128)
self.assertEqual(new_offset, offset)
if __name__ == "__main__":
unittest.main()
assert new_offset == offset
@@ -13,7 +13,6 @@ import os
import subprocess
import threading
import time
import unittest
import pytest
import torch
@@ -195,43 +194,43 @@ def _md5(tensor: torch.Tensor) -> str:
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestBasicConnector(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestBasicConnector:
"""Verify connector initialization, put, cleanup, and health check."""
def test_initialization(self):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=1024 * 1024)) as c:
self.assertNotEqual(c.rpc_port, 0)
self.assertEqual(c.pool_size, 1024 * 1024)
self.assertTrue(c.pool.is_pinned())
assert c.rpc_port != 0
assert c.pool_size == 1024 * 1024
assert c.pool.is_pinned()
health = c.health()
self.assertEqual(health["status"], "healthy")
assert health["status"] == "healthy"
def test_put_tensor_bytes_object(self):
"""Put tensor / bytes / dict and verify metadata."""
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
ok, sz, meta = c.put("s0", "s1", "t", torch.randn(100))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, sz, meta = c.put("s0", "s1", "b", b"hello" * 100)
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, sz, meta = c.put("s0", "s1", "d", {"k": [1, 2, 3]})
self.assertTrue(ok)
self.assertFalse(meta["is_fast_path"])
assert ok
assert not meta["is_fast_path"]
def test_cleanup_releases_buffer(self):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
c.put("s0", "s1", "r1", torch.randn(100))
key = MooncakeTransferEngineConnector._make_key("r1", "s0", "s1")
self.assertIn(key, c._local_buffers)
assert key in c._local_buffers
c.cleanup("r1", from_stage="s0", to_stage="s1")
self.assertNotIn(key, c._local_buffers)
assert key not in c._local_buffers
def test_pool_exhaustion_and_recovery(self):
"""Fill pool, verify failure, free, verify recovery."""
@@ -247,7 +246,7 @@ class TestBasicConnector(unittest.TestCase):
for rid in ids:
c.cleanup(rid, from_stage="s", to_stage="s")
ok, _, _ = c.put("s", "s", "recovery", torch.randn(1000))
self.assertTrue(ok, "Pool recovery failed after cleanup")
assert ok, "Pool recovery failed after cleanup"
# ---------------------------------------------------------------------------
@@ -255,8 +254,8 @@ class TestBasicConnector(unittest.TestCase):
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestEndToEnd(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestEndToEnd:
"""E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types."""
def _pair(self, pool_size=16 * 1024 * 1024):
@@ -269,14 +268,14 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = torch.randn(1024, 1024, dtype=torch.float32)
ok, sz, meta = p.put("s0", "s1", "t1", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "t1", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
self.assertIsInstance(buf, ManagedBuffer)
assert isinstance(buf, ManagedBuffer)
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertTrue(torch.equal(recon, orig))
assert torch.equal(recon, orig)
buf.release()
finally:
p.close()
@@ -287,15 +286,15 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = b"Hello RDMA! " * 1000
ok, _, meta = p.put("s0", "s1", "b1", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "b1", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recv = buf.to_bytes() if hasattr(buf, "to_bytes") else buf
if hasattr(buf, "release"):
buf.release()
self.assertEqual(recv, orig)
assert recv == orig
finally:
p.close()
c.close()
@@ -305,13 +304,13 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = {"msg": "hello", "nums": [1, 2, 3], "nested": {"k": "v"}}
ok, _, meta = p.put("s0", "s1", "o1", orig)
self.assertTrue(ok)
self.assertFalse(meta["is_fast_path"])
assert ok
assert not meta["is_fast_path"]
time.sleep(1.0)
result = c.get("s0", "s1", "o1", meta)
self.assertIsNotNone(result)
assert result is not None
recv, _ = result
self.assertEqual(recv, orig)
assert recv == orig
finally:
p.close()
c.close()
@@ -327,15 +326,15 @@ class TestEndToEnd(unittest.TestCase):
sbuf.as_tensor(dtype=torch.float32, shape=shape).copy_(ref)
ok, sz, meta = p.put("s0", "s1", "zc", sbuf)
self.assertTrue(ok)
self.assertTrue(meta.get("is_fast_path"))
assert ok
assert meta.get("is_fast_path")
time.sleep(0.5)
result = c.get("s0", "s1", "zc", meta)
self.assertIsNotNone(result)
assert result is not None
rbuf, _ = result
recon = rbuf.as_tensor(dtype=torch.float32, shape=shape)
self.assertTrue(torch.equal(recon, ref))
assert torch.equal(recon, ref)
rbuf.release()
finally:
p.close()
@@ -348,13 +347,13 @@ class TestEndToEnd(unittest.TestCase):
orig = torch.randn(5000, 5000, dtype=torch.float32)
md5_send = _md5(orig)
ok, _, meta = p.put("s0", "s1", "lg", orig)
self.assertTrue(ok)
assert ok
time.sleep(1.0)
result = c.get("s0", "s1", "lg", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertEqual(_md5(recon), md5_send)
assert _md5(recon) == md5_send
buf.release()
finally:
p.close()
@@ -374,23 +373,23 @@ class TestEndToEnd(unittest.TestCase):
time.sleep(0.5)
for name, data in cases:
ok, _, meta = p.put("s0", "s1", name, data)
self.assertTrue(ok, f"put failed: {name}")
assert ok, f"put failed: {name}"
time.sleep(0.3)
result = c.get("s0", "s1", name, meta)
self.assertIsNotNone(result, f"get failed: {name}")
assert result is not None, f"get failed: {name}"
recv, _ = result
if isinstance(data, torch.Tensor):
self.assertIsInstance(recv, ManagedBuffer)
assert isinstance(recv, ManagedBuffer)
recon = recv.as_tensor(dtype=data.dtype, shape=data.shape)
self.assertTrue(torch.equal(recon, data), f"mismatch: {name}")
assert torch.equal(recon, data), f"mismatch: {name}"
recv.release()
elif isinstance(data, bytes):
got = recv.to_bytes() if hasattr(recv, "to_bytes") else recv
if hasattr(recv, "release"):
recv.release()
self.assertEqual(got, data)
assert got == data
else:
self.assertEqual(recv, data)
assert recv == data
finally:
p.close()
c.close()
@@ -420,24 +419,24 @@ class TestEndToEnd(unittest.TestCase):
t.start()
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"errors: {errors}")
assert len(errors) == 0, f"errors: {errors}"
def test_auto_cleanup(self):
"""Producer buffer should be released after consumer get."""
p, c = self._pair()
try:
ok, _, meta = p.put("s0", "s1", "ac", torch.randn(100))
self.assertTrue(ok)
assert ok
key = MooncakeTransferEngineConnector._make_key("ac", "s0", "s1")
self.assertIn(key, p._local_buffers)
assert key in p._local_buffers
time.sleep(0.5)
result = c.get("s0", "s1", "ac", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
if isinstance(buf, ManagedBuffer):
buf.release()
time.sleep(0.3)
self.assertNotIn(key, p._local_buffers)
assert key not in p._local_buffers
finally:
p.close()
c.close()
@@ -448,22 +447,22 @@ class TestEndToEnd(unittest.TestCase):
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestLifecycle(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestLifecycle:
"""Close, context manager, double-close safety."""
def test_close_releases_resources(self):
c = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size=1024 * 1024))
c.put("s0", "s1", "x", torch.randn(100))
c.close()
self.assertTrue(c._stop_event.is_set())
self.assertEqual(len(c._local_buffers), 0)
assert c._stop_event.is_set()
assert len(c._local_buffers) == 0
def test_context_manager(self):
with MooncakeTransferEngineConnector(_connector_config(_free_port())) as c:
ok, _, _ = c.put("s0", "s1", "ctx", torch.randn(50))
self.assertTrue(ok)
self.assertTrue(c._stop_event.is_set())
assert ok
assert c._stop_event.is_set()
def test_double_close_safe(self):
c = MooncakeTransferEngineConnector(_connector_config(_free_port()))
@@ -477,9 +476,9 @@ class TestLifecycle(unittest.TestCase):
@pytest.mark.cuda
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
@unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
class TestGPUPool(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestGPUPool:
"""GPU memory pool: initialization, put (CPU/GPU tensor), E2E transfer."""
@staticmethod
@@ -488,18 +487,18 @@ class TestGPUPool(unittest.TestCase):
def test_gpu_pool_init(self):
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
self.assertEqual(c.pool_device, "cuda:0")
self.assertTrue(c.pool.is_cuda)
assert c.pool_device == "cuda:0"
assert c.pool.is_cuda
def test_gpu_pool_put_cpu_and_gpu_tensor(self):
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
ok, _, meta = c.put("s0", "s1", "h2d", torch.randn(256, 256))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, _, meta = c.put("s0", "s1", "d2d", torch.randn(256, 256, device="cuda:0"))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
def test_gpu_e2e_transfer(self):
p = MooncakeTransferEngineConnector(self._gpu_cfg(_free_port()))
@@ -507,14 +506,14 @@ class TestGPUPool(unittest.TestCase):
try:
orig = torch.randn(512, 512, dtype=torch.float32, device="cuda:0")
ok, _, meta = p.put("s0", "s1", "ge", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "ge", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertTrue(recon.is_cuda)
self.assertTrue(torch.equal(recon.cpu(), orig.cpu()))
assert recon.is_cuda
assert torch.equal(recon.cpu(), orig.cpu())
buf.release()
finally:
p.close()
@@ -527,8 +526,8 @@ class TestGPUPool(unittest.TestCase):
@pytest.mark.slow
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestStressCorrectness(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestStressCorrectness:
"""
Slow but high-value regression tests: concurrent put+get with data
integrity, large payload, edge cases, and sustained stress.
@@ -581,20 +580,20 @@ class TestStressCorrectness(unittest.TestCase):
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"Producer errors: {errors}")
self.assertEqual(len(meta_map), num_workers)
assert len(errors) == 0, f"Producer errors: {errors}"
assert len(meta_map) == num_workers
time.sleep(1.0)
# Consumer: get each tensor and verify integrity
for rid, meta in meta_map.items():
result = c.get("s0", "s1", rid, meta)
self.assertIsNotNone(result, f"get returned None for {rid}")
assert result is not None, f"get returned None for {rid}"
buf, _ = result
self.assertIsInstance(buf, ManagedBuffer, f"{rid} not ManagedBuffer")
assert isinstance(buf, ManagedBuffer), f"{rid} not ManagedBuffer"
recon = buf.as_tensor(dtype=torch.float32, shape=(512, 512))
recv_md5 = _md5(recon)
self.assertEqual(recv_md5, md5_sent[rid], f"MD5 mismatch for {rid}")
assert recv_md5 == md5_sent[rid], f"MD5 mismatch for {rid}"
buf.release()
finally:
p.close()
@@ -683,12 +682,8 @@ class TestStressCorrectness(unittest.TestCase):
for t in threads:
t.join()
ct.join(timeout=60)
self.assertEqual(len(errors), 0, f"errors: {errors}")
self.assertEqual(
consumed_count[0],
num_items,
f"Consumer only processed {consumed_count[0]}/{num_items} items",
)
assert len(errors) == 0, f"errors: {errors}"
assert consumed_count[0] == num_items, f"Consumer only processed {consumed_count[0]}/{num_items} items"
finally:
p.close()
c.close()
@@ -701,13 +696,13 @@ class TestStressCorrectness(unittest.TestCase):
try:
orig = torch.tensor([3.14], dtype=torch.float32)
ok, _, meta = p.put("s0", "s1", "tiny", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "tiny", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=torch.float32, shape=(1,))
self.assertTrue(torch.equal(recon, orig))
assert torch.equal(recon, orig)
buf.release()
finally:
p.close()
@@ -718,7 +713,7 @@ class TestStressCorrectness(unittest.TestCase):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=8 * 1024 * 1024)) as c:
ok, sz, meta = c.put("s0", "s1", "empty_b", b"")
self.assertFalse(ok, "Empty bytes should be rejected by connector")
assert not ok, "Empty bytes should be rejected by connector"
# -- Large payload stress (500 MB) --
@@ -731,13 +726,13 @@ class TestStressCorrectness(unittest.TestCase):
orig = torch.randn(side, side, dtype=torch.float32)
md5_send = _md5(orig)
ok, _, meta = p.put("s0", "s1", "500mb", orig)
self.assertTrue(ok)
assert ok
time.sleep(2.0)
result = c.get("s0", "s1", "500mb", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertEqual(_md5(recon), md5_send)
assert _md5(recon) == md5_send
buf.release()
finally:
p.close()
@@ -752,12 +747,8 @@ class TestStressCorrectness(unittest.TestCase):
for i in range(50):
rid = f"cycle_{i}"
ok, _, _ = c.put("s0", "s1", rid, torch.randn(256, 256))
self.assertTrue(ok, f"Put failed at iteration {i}")
assert ok, f"Put failed at iteration {i}"
c.cleanup(rid, from_stage="s0", to_stage="s1")
# After all cycles, pool should be fully recovered
ok, _, _ = c.put("s0", "s1", "final", torch.randn(1024, 1024))
self.assertTrue(ok, "Pool not fully recovered after rapid cycles")
if __name__ == "__main__":
unittest.main()
assert ok, "Pool not fully recovered after rapid cycles"
@@ -48,7 +48,6 @@ def inference(model_name: str, offload: bool = True):
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
@pytest.mark.skipif(current_omni_platform.is_npu() or current_omni_platform.is_rocm(), reason="Hardware not supported")
@pytest.mark.parametrize("model_name", models)
def test_cpu_offload_diffusion_model(model_name: str):
try:
@@ -30,7 +30,7 @@ def run_inference(
) -> float:
# For now, only support on GPU, so apply torch.cuda operations here
# NPU / ROCm platforms are expected to be detected and skipped this test function
torch.cuda.empty_cache()
current_omni_platform.empty_cache()
device_index = torch.cuda.current_device()
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
@@ -68,7 +68,6 @@ def run_inference(
return peak
@pytest.mark.skipif(current_omni_platform.is_npu() or current_omni_platform.is_rocm(), reason="Hardware not supported")
@pytest.mark.parametrize("model_name", MODELS_SAVED_MEMORY_MB.keys())
def test_layerwise_offload_diffusion_model(model_name: str):
"""Test that layerwise offloading reduces GPU memory usage.
+51 -49
View File
@@ -12,11 +12,11 @@ Strategy:
and correctness of the model definition itself, independent of the pipeline mocks.
"""
from unittest.mock import MagicMock, patch
import pytest
import torch
from pytest_mock import MockerFixture
from tests.utils import hardware_test
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
@@ -29,15 +29,15 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@pytest.fixture
def mock_dependencies(monkeypatch):
def mock_dependencies(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
"""
Mock external dependencies to avoid loading real models.
"""
device = get_local_device()
# Mock Tokenizer
mock_tokenizer = MagicMock()
mock_tokenizer.return_value = MagicMock(
mock_tokenizer = mocker.MagicMock()
mock_tokenizer.return_value = mocker.MagicMock(
input_ids=torch.zeros((1, 50), dtype=torch.long, device=device),
attention_mask=torch.ones((1, 50), dtype=torch.long, device=device),
)
@@ -45,13 +45,13 @@ def mock_dependencies(monkeypatch):
mock_tokenizer.model_max_length = 1024
# Mock Text Encoder
mock_text_encoder = MagicMock()
mock_text_encoder = mocker.MagicMock()
mock_text_encoder.dtype = torch.float32
# Output of text encoder must be on the same device as inputs (which are moved to execution_device)
mock_text_encoder.return_value.last_hidden_state = torch.randn(1, 50, 32, device=device)
# Mock VAE
mock_vae = MagicMock()
mock_vae = mocker.MagicMock()
mock_vae.config.block_out_channels = [128, 256, 512, 512] # Scale factor 8
mock_vae.config.scale_factor_temporal = 1
mock_vae.config.scale_factor_spatial = 8
@@ -63,8 +63,8 @@ def mock_dependencies(monkeypatch):
mock_vae.to.return_value = mock_vae
# Mock Scheduler
mock_scheduler = MagicMock()
mock_scheduler.config = MagicMock()
mock_scheduler = mocker.MagicMock()
mock_scheduler.config = mocker.MagicMock()
# Timesteps on device to match latents during denoising loop interaction if needed
mock_scheduler.timesteps = torch.tensor([1.0, 0.5, 0.0], device=device)
mock_scheduler.set_timesteps.return_value = None
@@ -95,7 +95,7 @@ def mock_dependencies(monkeypatch):
@pytest.fixture
def ovis_pipeline(mock_dependencies, monkeypatch):
def ovis_pipeline(mocker: MockerFixture, mock_dependencies, monkeypatch: pytest.MonkeyPatch):
"""
Creates an OvisImagePipeline instance with mocked components.
"""
@@ -122,8 +122,8 @@ def ovis_pipeline(mock_dependencies, monkeypatch):
# Mock Transformer Layer separately to avoid full init
# We patch OvisImageTransformer2DModel class in the module
mock_transformer_cls = MagicMock()
mock_transformer_instance = MagicMock()
mock_transformer_cls = mocker.MagicMock()
mock_transformer_instance = mocker.MagicMock()
mock_transformer_instance.dtype = torch.float32
mock_transformer_instance.in_channels = 16 # Must be 16 so num_channel_latents=4, packed=16
# Forward return: noise prediction
@@ -144,8 +144,8 @@ def ovis_pipeline(mock_dependencies, monkeypatch):
# Initialize pipeline
# We use a dummy model path check override
with patch("os.path.exists", return_value=True):
pipeline = OvisImagePipeline(od_config=od_config)
mocker.patch("os.path.exists", return_value=True)
pipeline = OvisImagePipeline(od_config=od_config)
return pipeline
@@ -223,9 +223,11 @@ def test_resolution_check(ovis_pipeline):
assert output is not None
def test_real_transformer_init_and_forward():
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
def test_real_transformer_init_and_forward(mocker: MockerFixture):
"""Test the real OvisImageTransformer2DModel initialization and forward pass for coverage."""
from unittest.mock import patch
from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel
@@ -249,42 +251,42 @@ def test_real_transformer_init_and_forward():
# Mock distributed state for QKVParallelLinear initialization
# We patch get_tp_group because get_tensor_model_parallel_rank calls it and asserts _TP is not None
mock_group = MagicMock()
mock_group = mocker.MagicMock()
mock_group.rank_in_group = 0
mock_group.world_size = 1
with patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group):
# Initialize real model
model = OvisImageTransformer2DModel(
od_config=od_config,
patch_size=1,
in_channels=16,
out_channels=16,
num_single_layers=1,
attention_head_dim=8,
num_attention_heads=2,
joint_attention_dim=32,
axes_dims_rope=(2, 2, 4),
).to(device)
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
# Initialize real model
model = OvisImageTransformer2DModel(
od_config=od_config,
patch_size=1,
in_channels=16,
out_channels=16,
num_single_layers=1,
attention_head_dim=8,
num_attention_heads=2,
joint_attention_dim=32,
axes_dims_rope=(2, 2, 4),
).to(device)
# Create dummy inputs
B, Seq, C = 1, 16, 16
hidden_states = torch.randn(B, Seq, C, device=device)
encoder_hidden_states = torch.randn(B, 10, 32, device=device) # joint_attention_dim=32
timestep = torch.tensor([1], device=device)
img_ids = torch.zeros(Seq, 3, device=device)
txt_ids = torch.zeros(10, 3, device=device)
# Create dummy inputs
B, Seq, C = 1, 16, 16
hidden_states = torch.randn(B, Seq, C, device=device)
encoder_hidden_states = torch.randn(B, 10, 32, device=device) # joint_attention_dim=32
timestep = torch.tensor([1], device=device)
img_ids = torch.zeros(Seq, 3, device=device)
txt_ids = torch.zeros(10, 3, device=device)
# Run forward
output = model(
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
timestep=timestep,
img_ids=img_ids,
txt_ids=txt_ids,
return_dict=False,
)
# Run forward
output = model(
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
timestep=timestep,
img_ids=img_ids,
txt_ids=txt_ids,
return_dict=False,
)
assert output is not None
assert isinstance(output, tuple)
assert output[0].shape == hidden_states.shape
assert output is not None
assert isinstance(output, tuple)
assert output[0].shape == hidden_states.shape
@@ -71,7 +71,7 @@ def get_system_prompt():
def get_prompt(prompt_type="text_only"):
prompts = {
"text_only": "What is the capital of China?",
"text_only": "What is the capital of China? Answer in 20 words.",
"mix": "What is recited in the audio? What is in this image? What is in this video?",
"text_video": "What is in this video? ",
"text_image": "What is in this image? ",
@@ -22,6 +22,7 @@ stage_args:
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
@@ -0,0 +1,31 @@
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 16384
max_num_batched_tokens: 16384
max_num_seqs: 1
gpu_memory_utilization: 0.9
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -1,31 +0,0 @@
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0,1"
max_batch_size: 5
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 2
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
+3 -3
View File
@@ -15,8 +15,8 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
SEED = 42
stage_config = str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
stage_config = str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_thinker_ci.yaml")
model = "Qwen/Qwen2.5-Omni-7B"
async def generate(
@@ -60,7 +60,7 @@ async def generate(
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.asyncio
async def test_abort():
with ExitStack() as after:
@@ -10,11 +10,11 @@ OpenAI-compatible async text-to-image generation API endpoints in api_server.py.
import base64
import io
from argparse import Namespace
from unittest.mock import AsyncMock, Mock
import pytest
from fastapi.testclient import TestClient
from PIL import Image
from pytest_mock import MockerFixture
from vllm import SamplingParams
from vllm_omni.entrypoints.openai.image_api_utils import (
@@ -128,11 +128,11 @@ class FakeAsyncOmni:
@pytest.fixture
def mock_async_diffusion():
def mock_async_diffusion(mocker: MockerFixture):
"""Mock AsyncOmniDiffusion instance that returns fake images"""
mock = Mock()
mock = mocker.Mock()
mock.is_running = True # For health endpoint
mock.check_health = AsyncMock() # For LLM mode health check
mock.check_health = mocker.AsyncMock() # For LLM mode health check
async def generate(**kwargs):
# Return n PIL images wrapped in result object
@@ -143,7 +143,7 @@ def mock_async_diffusion():
images = [Image.new("RGB", (64, 64), color="blue") for _ in range(n)]
return MockGenerationResult(images)
mock.generate = AsyncMock(side_effect=generate)
mock.generate = mocker.AsyncMock(side_effect=generate)
return mock
@@ -6,27 +6,26 @@ Tests that standard OpenAI API parameters (max_tokens, temperature, etc.)
are correctly applied to the comprehension stage while preserving YAML defaults.
"""
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm.sampling_params import SamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def mock_comprehension_stage():
def mock_comprehension_stage(mocker: MockerFixture):
"""Create a mock comprehension stage with is_comprehension=True."""
stage = MagicMock()
stage = mocker.MagicMock()
stage.is_comprehension = True
stage.model_stage = "comprehension"
return stage
@pytest.fixture
def mock_other_stage():
def mock_other_stage(mocker: MockerFixture):
"""Create a mock non-comprehension stage."""
stage = MagicMock()
stage = mocker.MagicMock()
stage.is_comprehension = False
stage.model_stage = "other"
return stage
@@ -57,9 +56,15 @@ def default_other_params():
@pytest.fixture
def mock_engine_client(mock_comprehension_stage, mock_other_stage, default_comprehension_params, default_other_params):
def mock_engine_client(
mock_comprehension_stage,
mock_other_stage,
default_comprehension_params,
default_other_params,
mocker: MockerFixture,
):
"""Create mock engine client with stage_list and default_sampling_params_list."""
engine_client = MagicMock()
engine_client = mocker.MagicMock()
engine_client.stage_list = [mock_comprehension_stage, mock_other_stage]
engine_client.default_sampling_params_list = [
default_comprehension_params,
@@ -80,9 +85,9 @@ def serving_chat(mock_engine_client):
@pytest.fixture
def mock_request():
def mock_request(mocker: MockerFixture):
"""Create a mock request with all OpenAI sampling params set to None."""
request = MagicMock()
request = mocker.MagicMock()
# OpenAI standard sampling fields
request.temperature = None
request.top_p = None
@@ -294,35 +299,35 @@ def test_get_comprehension_stage_index_finds_first_stage(mock_engine_client):
assert instance._get_comprehension_stage_index() == 0
def test_get_comprehension_stage_index_finds_second_stage():
def test_get_comprehension_stage_index_finds_second_stage(mocker: MockerFixture):
"""Test finding comprehension stage when it's at index 1."""
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
instance = object.__new__(OmniOpenAIServingChat)
other = MagicMock()
other = mocker.MagicMock()
other.is_comprehension = False
comprehension = MagicMock()
comprehension = mocker.MagicMock()
comprehension.is_comprehension = True
instance.engine_client = MagicMock()
instance.engine_client = mocker.MagicMock()
instance.engine_client.stage_list = [other, comprehension]
assert instance._get_comprehension_stage_index() == 1
def test_get_comprehension_stage_index_raises_when_not_found():
def test_get_comprehension_stage_index_raises_when_not_found(mocker: MockerFixture):
"""Test that ValueError is raised when no comprehension stage exists."""
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
instance = object.__new__(OmniOpenAIServingChat)
stage1 = MagicMock()
stage1 = mocker.MagicMock()
stage1.is_comprehension = False
stage2 = MagicMock()
stage2 = mocker.MagicMock()
stage2.is_comprehension = False
instance.engine_client = MagicMock()
instance.engine_client = mocker.MagicMock()
instance.engine_client.stage_list = [stage1, stage2]
with pytest.raises(ValueError, match="No comprehension stage"):
@@ -1,13 +1,14 @@
# tests/entrypoints/openai/test_serving_speech.py
import logging
from inspect import Signature, signature
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pydantic import ValidationError
from pytest_mock import MockerFixture
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio, OpenAICreateSpeechRequest
@@ -26,25 +27,24 @@ class TestAudioMixin:
def audio_mixin(self):
return AudioMixin()
def test_stereo_to_mono_conversion(self, audio_mixin):
def test_stereo_to_mono_conversion(self, audio_mixin, mocker: MockerFixture):
stereo_tensor = np.random.rand(24000, 2).astype(np.float32)
audio_obj = CreateAudio(audio_tensor=stereo_tensor)
with (
patch.object(
audio_mixin, "_apply_speed_adjustment", side_effect=lambda tensor, speed, sr: (tensor, sr)
) as mock_speed,
patch("soundfile.write") as _,
):
audio_mixin.create_audio(audio_obj)
mock_speed = mocker.patch.object(
audio_mixin, "_apply_speed_adjustment", side_effect=lambda tensor, speed, sr: (tensor, sr)
)
mocker.patch("soundfile.write")
# Check that the tensor passed to speed adjustment is mono
mock_speed.assert_called_once()
adjusted_tensor = mock_speed.call_args[0][0]
assert len(adjusted_tensor) == 24000
audio_mixin.create_audio(audio_obj)
@patch("librosa.effects.time_stretch")
def test_speed_adjustment(self, mock_time_stretch, audio_mixin):
# Check that the tensor passed to speed adjustment is mono
mock_speed.assert_called_once()
adjusted_tensor = mock_speed.call_args[0][0]
assert len(adjusted_tensor) == 24000
def test_speed_adjustment(self, audio_mixin, mocker: MockerFixture):
mock_time_stretch = mocker.patch("librosa.effects.time_stretch")
mock_time_stretch.return_value = np.zeros(12000)
audio_tensor = np.random.rand(24000).astype(np.float32)
@@ -53,8 +53,8 @@ class TestAudioMixin:
mock_time_stretch.assert_called_with(y=audio_tensor, rate=2.0)
assert adjusted_audio.shape == (12000,)
@patch("soundfile.write")
def test_unsupported_format_fallback(self, mock_write, audio_mixin, caplog):
def test_unsupported_format_fallback(self, audio_mixin, caplog, mocker: MockerFixture):
mock_write = mocker.patch("soundfile.write")
audio_tensor = np.random.rand(24000).astype(np.float32)
# Use a format that is not in the list of supported formats
audio_obj = CreateAudio(audio_tensor=audio_tensor, response_format="vorbis")
@@ -66,50 +66,50 @@ class TestAudioMixin:
write_kwargs = mock_write.call_args.kwargs
assert write_kwargs["format"] == "WAV"
def test_mono_audio_preservation(self, audio_mixin):
def test_mono_audio_preservation(self, audio_mixin, mocker: MockerFixture):
"""Test that mono (1D) audio tensors are processed correctly and passed to writer."""
mono_tensor = np.random.rand(24000).astype(np.float32)
audio_obj = CreateAudio(audio_tensor=mono_tensor)
with patch("soundfile.write") as mock_write:
audio_mixin.create_audio(audio_obj)
mock_write = mocker.patch("soundfile.write")
audio_mixin.create_audio(audio_obj)
mock_write.assert_called_once()
# Verify the tensor passed to soundfile.write is the exact 1D tensor
output_tensor = mock_write.call_args[0][1]
assert output_tensor.ndim == 1
assert output_tensor.shape == (24000,)
assert np.array_equal(output_tensor, mono_tensor)
mock_write.assert_called_once()
# Verify the tensor passed to soundfile.write is the exact 1D tensor
output_tensor = mock_write.call_args[0][1]
assert output_tensor.ndim == 1
assert output_tensor.shape == (24000,)
assert np.array_equal(output_tensor, mono_tensor)
def test_stereo_audio_preservation(self, audio_mixin):
def test_stereo_audio_preservation(self, audio_mixin, mocker: MockerFixture):
"""Test that stereo (2D) audio tensors are processed correctly and preserved."""
stereo_tensor = np.random.rand(24000, 2).astype(np.float32)
audio_obj = CreateAudio(audio_tensor=stereo_tensor)
with patch("soundfile.write") as mock_write:
audio_mixin.create_audio(audio_obj)
mock_write = mocker.patch("soundfile.write")
audio_mixin.create_audio(audio_obj)
mock_write.assert_called_once()
# Verify the tensor passed to soundfile.write is the exact 2D tensor
output_tensor = mock_write.call_args[0][1]
assert output_tensor.ndim == 2
assert output_tensor.shape == (24000, 2)
assert np.array_equal(output_tensor, stereo_tensor)
mock_write.assert_called_once()
# Verify the tensor passed to soundfile.write is the exact 2D tensor
output_tensor = mock_write.call_args[0][1]
assert output_tensor.ndim == 2
assert output_tensor.shape == (24000, 2)
assert np.array_equal(output_tensor, stereo_tensor)
def test_speed_adjustment_bypass(self, audio_mixin):
def test_speed_adjustment_bypass(self, audio_mixin, mocker: MockerFixture):
"""Test that speed=1.0 bypasses the expensive librosa time stretching."""
audio_tensor = np.random.rand(24000).astype(np.float32)
with patch("librosa.effects.time_stretch") as mock_time_stretch:
# speed=1.0 should return immediately without calling librosa
result, _ = audio_mixin._apply_speed_adjustment(audio_tensor, speed=1.0, sample_rate=24000)
mock_time_stretch = mocker.patch("librosa.effects.time_stretch")
# speed=1.0 should return immediately without calling librosa
result, _ = audio_mixin._apply_speed_adjustment(audio_tensor, speed=1.0, sample_rate=24000)
mock_time_stretch.assert_not_called()
assert np.array_equal(result, audio_tensor)
mock_time_stretch.assert_not_called()
assert np.array_equal(result, audio_tensor)
@patch("librosa.effects.time_stretch")
def test_speed_adjustment_stereo_handling(self, mock_time_stretch, audio_mixin):
def test_speed_adjustment_stereo_handling(self, audio_mixin, mocker: MockerFixture):
"""Test that speed adjustment is attempted on stereo inputs."""
mock_time_stretch = mocker.patch("librosa.effects.time_stretch")
stereo_tensor = np.random.rand(24000, 2).astype(np.float32)
# Mock return value representing a sped-up version (half length)
mock_time_stretch.return_value = np.zeros((12000, 2), dtype=np.float32)
@@ -161,22 +161,22 @@ def create_mock_audio_output_for_test(
@pytest.fixture
def test_app():
def test_app(mocker: MockerFixture):
# Mock the engine client
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
async def mock_generate_fn(*args, **kwargs):
yield create_mock_audio_output_for_test(request_id=kwargs.get("request_id"))
mock_engine_client.generate = MagicMock(side_effect=mock_generate_fn)
mock_engine_client.generate = mocker.MagicMock(side_effect=mock_generate_fn)
mock_engine_client.default_sampling_params_list = [{}]
# Mock models to have an is_base_model method
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
mock_request_logger = MagicMock()
mock_request_logger = mocker.MagicMock()
speech_server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
@@ -186,7 +186,7 @@ def test_app():
# Patch the signature of create_speech to remove 'raw_request' for FastAPI route introspection
original_create_speech = speech_server.create_speech
_ = MagicMock(side_effect=original_create_speech)
_ = mocker.MagicMock(side_effect=original_create_speech)
sig = signature(original_create_speech)
@@ -253,11 +253,13 @@ class TestSpeechAPI:
response = client.post("/v1/audio/speech", json=payload)
assert response.status_code == 422 # Unprocessable Entity
@patch("vllm_omni.entrypoints.openai.serving_speech.OmniOpenAIServingSpeech.create_audio")
def test_speed_parameter_is_used(self, mock_create_audio, test_app):
def test_speed_parameter_is_used(self, test_app, mocker: MockerFixture):
mock_create_audio = mocker.patch(
"vllm_omni.entrypoints.openai.serving_speech.OmniOpenAIServingSpeech.create_audio"
)
client = TestClient(test_app)
mock_audio_response = MagicMock()
mock_audio_response = mocker.MagicMock()
mock_audio_response.audio_data = b"dummy_audio"
mock_audio_response.media_type = "audio/wav"
mock_create_audio.return_value = mock_audio_response
@@ -287,17 +289,17 @@ class TestTTSMethods:
"""Unit tests for TTS validation and parameter building."""
@pytest.fixture
def speech_server(self):
mock_engine_client = MagicMock()
def speech_server(self, mocker: MockerFixture):
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
mock_engine_client.tts_max_instructions_length = None
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
def test_is_tts_detection_no_stage(self, speech_server):
@@ -306,25 +308,25 @@ class TestTTSMethods:
assert speech_server._is_tts is False
assert speech_server._tts_stage is None
def test_is_tts_detection_with_tts_stage(self):
def test_is_tts_detection_with_tts_stage(self, mocker: MockerFixture):
"""Test TTS model detection when TTS stage exists."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.tts_max_instructions_length = None
# Create a TTS stage
mock_stage = MagicMock()
mock_stage = mocker.MagicMock()
mock_stage.model_stage = "qwen3_tts"
mock_stage.tts_args = {}
mock_engine_client.stage_list = [mock_stage]
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
assert server._is_tts is True
@@ -379,24 +381,24 @@ class TestTTSMethods:
assert params["language"] == ["English"]
assert params["task_type"] == ["CustomVoice"]
def test_load_supported_speakers(self):
def test_load_supported_speakers(self, mocker: MockerFixture):
"""Test _load_supported_speakers."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
# Mock talker_config with mixed-case speaker names
mock_talker_config = MagicMock()
mock_talker_config = mocker.MagicMock()
mock_talker_config.spk_id = {"Ryan": 0, "Vivian": 1, "Aiden": 2}
mock_engine_client.model_config.hf_config.talker_config = mock_talker_config
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
# Verify speakers are normalized to lowercase
@@ -407,34 +409,34 @@ class TestTTSMethods:
# Fixture creates server with no CLI override and no TTS stage
assert speech_server._max_instructions_length == 500
def test_max_instructions_length_cli_override(self):
def test_max_instructions_length_cli_override(self, mocker: MockerFixture):
"""Test CLI override (stored in engine_client) takes highest priority."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
# CLI override is stored in engine_client
mock_engine_client.tts_max_instructions_length = 1000
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
# Value is cached during __init__
assert server._max_instructions_length == 1000
def test_max_instructions_length_stage_config(self):
def test_max_instructions_length_stage_config(self, mocker: MockerFixture):
"""Test stage config value is used when no CLI override."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.tts_max_instructions_length = None # No CLI override
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
# Mock stage with tts_args
mock_stage = MagicMock()
mock_stage = mocker.MagicMock()
mock_stage.model_stage = "qwen3_tts"
mock_stage.tts_args = {"max_instructions_length": 750}
mock_engine_client.stage_list = [mock_stage]
@@ -442,22 +444,22 @@ class TestTTSMethods:
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
# Value is cached during __init__
assert server._max_instructions_length == 750
def test_max_instructions_length_cli_overrides_stage_config(self):
def test_max_instructions_length_cli_overrides_stage_config(self, mocker: MockerFixture):
"""Test CLI override (in engine_client) takes precedence over stage config."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
# CLI override stored in engine_client
mock_engine_client.tts_max_instructions_length = 2000
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
# Mock stage with tts_args
mock_stage = MagicMock()
mock_stage = mocker.MagicMock()
mock_stage.model_stage = "qwen3_tts"
mock_stage.tts_args = {"max_instructions_length": 750}
mock_engine_client.stage_list = [mock_stage]
@@ -465,25 +467,25 @@ class TestTTSMethods:
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
# CLI value (2000) should override stage config (750)
assert server._max_instructions_length == 2000
def test_validate_instructions_length_uses_cached_value(self):
def test_validate_instructions_length_uses_cached_value(self, mocker: MockerFixture):
"""Test instructions length validation uses cached _max_instructions_length."""
mock_engine_client = MagicMock()
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
# CLI override with max length of 10 characters
mock_engine_client.tts_max_instructions_length = 10
mock_models = MagicMock()
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
request_logger=mocker.MagicMock(),
)
# Verify cached value
@@ -506,3 +508,110 @@ class TestTTSMethods:
error = server._validate_tts_request(req)
assert error is not None
assert "max 10 characters" in error
class TestStreamingProtocolValidation:
"""Unit tests for the stream field validators in OpenAICreateSpeechRequest."""
def test_stream_validation_errors(self):
"""stream=True requires response_format='pcm' and speed=1.0."""
with pytest.raises(ValidationError, match="response_format='pcm'"):
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav")
with pytest.raises(ValidationError, match="Speed adjustment is not supported"):
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm", speed=2.0)
def test_stream_valid(self):
"""stream=True + response_format='pcm' + speed=1.0 is accepted."""
req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm")
assert req.stream is True
def test_sse_stream_format_is_blocked(self):
"""stream_format='sse' is blocked."""
with pytest.raises(ValidationError, match="sse"):
OpenAICreateSpeechRequest(input="Hello", stream_format="sse")
class TestStreamingResponse:
"""Integration tests for the streaming audio response path."""
@pytest.fixture
def streaming_app(self, mocker: MockerFixture):
"""Test app whose mock engine yields one intermediate chunk then a final chunk."""
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
def _make_output(finished: bool) -> OmniRequestOutput:
chunk = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, 24000))
class MockCompletionOutput:
def __init__(self, index: int = 0):
self.index = index
self.text = ""
self.token_ids = []
self.finish_reason = "stop"
self.stop_reason = None
self.logprobs = None
class MockRequestOutput:
def __init__(self, audio_tensor: torch.Tensor):
self.request_id = "speech-stream-test"
self.outputs = [MockCompletionOutput(index=0)]
self.multimodal_output = {"audio": audio_tensor}
self.finished = finished
self.prompt_token_ids = None
self.encoder_prompt_token_ids = None
self.num_cached_tokens = None
self.prompt_logprobs = None
self.kv_transfer_params = None
return OmniRequestOutput(
stage_id=0,
final_output_type="audio",
request_output=MockRequestOutput(audio_tensor=chunk),
finished=finished,
)
async def mock_generate_streaming(*args, **kwargs):
yield _make_output(finished=False)
yield _make_output(finished=True)
mock_engine_client.generate = mocker.MagicMock(side_effect=mock_generate_streaming)
mock_engine_client.default_sampling_params_list = [{}]
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
speech_server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=mocker.MagicMock(),
)
original_create_speech = speech_server.create_speech
sig = signature(original_create_speech)
new_parameters = [p for name, p in sig.parameters.items() if name != "raw_request"]
new_sig = Signature(parameters=new_parameters, return_annotation=sig.return_annotation)
async def awaitable_create_speech(*args, **kwargs):
return await original_create_speech(*args, **kwargs)
awaitable_create_speech.__signature__ = new_sig
speech_server.create_speech = awaitable_create_speech
app = FastAPI()
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
return app
def test_streaming(self, streaming_app):
"""stream=True must return audio/pcm with non-empty body."""
client = TestClient(streaming_app)
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": "pcm"})
assert response.status_code == 200
assert "audio/pcm" in response.headers["content-type"]
assert len(response.content) > 0
def test_non_streaming_unchanged(self, streaming_app):
"""Non-streaming path must still return audio/wav."""
client = TestClient(streaming_app)
response = client.post("/v1/audio/speech", json={"input": "Hello", "response_format": "wav"})
assert response.status_code == 200
assert "audio/wav" in response.headers["content-type"]
@@ -5,18 +5,20 @@ Unit tests for OpenAI-compatible video generation endpoints.
"""
import io
from unittest.mock import patch
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from PIL import Image
from pydantic import ValidationError
from pytest_mock import MockerFixture
from vllm_omni.entrypoints.openai.api_server import router
from vllm_omni.entrypoints.openai.protocol.videos import VideoGenerationRequest, VideoResponseFormat
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class MockVideoResult:
def __init__(self, videos):
@@ -55,27 +57,27 @@ def _make_test_image_bytes(size=(64, 64)) -> bytes:
return buf.getvalue()
def test_t2v_video_generation_form(test_client):
def test_t2v_video_generation_form(test_client, mocker: MockerFixture):
fps_values = []
def _fake_encode(video, fps):
fps_values.append(fps)
return "Zg=="
with patch(
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
side_effect=_fake_encode,
):
response = test_client.post(
"/v1/videos",
data={
"prompt": "A cat runs across the street.",
"size": "640x360",
"seconds": "2",
"fps": "12",
"n": "2",
},
)
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "A cat runs across the street.",
"size": "640x360",
"seconds": "2",
"fps": "12",
"n": "2",
},
)
assert response.status_code == 200
data = response.json()
@@ -92,18 +94,18 @@ def test_t2v_video_generation_form(test_client):
assert fps_values == [12, 12]
def test_i2v_video_generation_form(test_client):
def test_i2v_video_generation_form(test_client, mocker: MockerFixture):
image_bytes = _make_test_image_bytes((48, 32))
with patch(
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
):
response = test_client.post(
"/v1/videos",
data={"prompt": "A bear playing with yarn."},
files={"input_reference": ("input.png", image_bytes, "image/png")},
)
)
response = test_client.post(
"/v1/videos",
data={"prompt": "A bear playing with yarn."},
files={"input_reference": ("input.png", image_bytes, "image/png")},
)
assert response.status_code == 200
data = response.json()
@@ -119,24 +121,24 @@ def test_i2v_video_generation_form(test_client):
assert input_image.size == (48, 32)
def test_seconds_defaults_fps_and_frames(test_client):
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
fps_values = []
def _fake_encode(video, fps):
fps_values.append(fps)
return "Zg=="
with patch(
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
side_effect=_fake_encode,
):
response = test_client.post(
"/v1/videos",
data={
"prompt": "A bird flying.",
"seconds": "3",
},
)
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "A bird flying.",
"seconds": "3",
},
)
assert response.status_code == 200
engine = test_client.app.state.openai_serving_video._engine_client
@@ -146,18 +148,18 @@ def test_seconds_defaults_fps_and_frames(test_client):
assert fps_values == [24]
def test_size_param_sets_width_height(test_client):
with patch(
def test_size_param_sets_width_height(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
):
response = test_client.post(
"/v1/videos",
data={
"prompt": "size test",
"size": "320x240",
},
)
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "size test",
"size": "320x240",
},
)
assert response.status_code == 200
engine = test_client.app.state.openai_serving_video._engine_client
@@ -166,23 +168,23 @@ def test_size_param_sets_width_height(test_client):
assert captured.height == 240
def test_sampling_params_pass_through(test_client):
with patch(
def test_sampling_params_pass_through(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
):
response = test_client.post(
"/v1/videos",
data={
"prompt": "param pass",
"num_inference_steps": "30",
"guidance_scale": "6.5",
"guidance_scale_2": "8.0",
"true_cfg_scale": "4.0",
"boundary_ratio": "0.7",
"flow_shift": "0.25",
},
)
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "param pass",
"num_inference_steps": "30",
"guidance_scale": "6.5",
"guidance_scale_2": "8.0",
"true_cfg_scale": "4.0",
"boundary_ratio": "0.7",
"flow_shift": "0.25",
},
)
assert response.status_code == 200
engine = test_client.app.state.openai_serving_video._engine_client
@@ -264,19 +266,19 @@ def test_invalid_n_raises_validation_error(test_client):
)
def test_negative_prompt_and_seed_pass_through(test_client):
with patch(
def test_negative_prompt_and_seed_pass_through(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
return_value="Zg==",
):
response = test_client.post(
"/v1/videos",
data={
"prompt": "snowy mountain",
"negative_prompt": "blurry",
"seed": "123",
},
)
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "snowy mountain",
"negative_prompt": "blurry",
"seed": "123",
},
)
assert response.status_code == 200
engine = test_client.app.state.openai_serving_video._engine_client
+108 -94
View File
@@ -2,9 +2,9 @@ import uuid
import warnings
from queue import Empty, Queue
from typing import Any
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -152,7 +152,7 @@ class _FakeZmqQueue:
class _FakeStage:
"""Lightweight Stage stub for multi-process pipeline version with queue support."""
def __init__(self, config, stage_init_timeout: int = 300):
def __init__(self, mocker: MockerFixture, config, stage_init_timeout: int = 300):
# Handle both dict and object configs
if isinstance(config, dict):
config = _FakeStageConfig(config)
@@ -178,6 +178,10 @@ class _FakeStage:
self._out_q = None
self._proc = None # Mock process reference
self._stage_init_timeout = max(0, int(stage_init_timeout))
# NOTE: pytest mocker fixture object is injected into test functions and classes
# with specific naming (e.g. `class TestXXX`, `def test_XXX` ) automatically,
# but **must be provided** when initializing this helper cls
self._mocker = mocker
def attach_queues(self, in_q, out_q):
"""Attach input and output queues."""
@@ -196,11 +200,11 @@ class _FakeStage:
):
"""Mock init_stage_worker: don't start real process, just send stage_ready message."""
# Create a mock process object
self._proc = MagicMock()
self._proc.start = MagicMock()
self._proc.join = MagicMock()
self._proc.is_alive = MagicMock(return_value=False)
self._proc.terminate = MagicMock()
self._proc = self._mocker.MagicMock()
self._proc.start = self._mocker.MagicMock()
self._proc.join = self._mocker.MagicMock()
self._proc.is_alive = self._mocker.MagicMock(return_value=False)
self._proc.terminate = self._mocker.MagicMock()
# Send stage_ready message to output queue
if self._out_q is not None:
try:
@@ -264,21 +268,23 @@ def fake_stage_config():
}
def _setup_engine_mocks(monkeypatch):
"""Helper function to set up common engine mocks."""
fake_engine = MagicMock()
def _setup_engine_mocks(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Helper function to set up common engine mocks.
fixture objects `monkeypatch` and `mocker` must be passed in.
"""
fake_engine = mocker.MagicMock()
# Add necessary attributes to fake_engine
fake_engine.tokenizer = MagicMock()
fake_engine.tokenizer = mocker.MagicMock()
fake_engine.log_stats = False
fake_engine.vllm_config = MagicMock()
fake_engine.vllm_config.model_config = MagicMock()
fake_engine.vllm_config = mocker.MagicMock()
fake_engine.vllm_config.model_config = mocker.MagicMock()
fake_engine.vllm_config.model_config.io_processor_plugin = None
fake_engine.get_supported_tasks = MagicMock(return_value=[])
fake_engine.model_config = MagicMock()
fake_engine.get_supported_tasks = mocker.MagicMock(return_value=[])
fake_engine.model_config = mocker.MagicMock()
fake_engine.model_config.io_processor_plugin = None
# Add registry with resolve_model_cls method
fake_registry = MagicMock()
fake_registry.resolve_model_cls = MagicMock(return_value=(MagicMock(), "test_arch"))
fake_registry = mocker.MagicMock()
fake_registry.resolve_model_cls = mocker.MagicMock(return_value=(mocker.MagicMock(), "test_arch"))
fake_engine.model_config.registry = fake_registry
fake_engine.vllm_config.model_config.registry = fake_registry
@@ -327,21 +333,23 @@ def _setup_engine_mocks(monkeypatch):
)
def _setup_multiprocessing_mocks(monkeypatch):
"""Helper function to set up multiprocessing mocks."""
def _setup_multiprocessing_mocks(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Helper function to set up multiprocessing mocks.
fixture objects `monkeypatch` and `mocker` must be passed in.
"""
import multiprocessing as mp
# Mock Process
fake_process_class = MagicMock()
fake_process_instance = MagicMock()
fake_process_instance.start = MagicMock()
fake_process_instance.join = MagicMock()
fake_process_instance.is_alive = MagicMock(return_value=False)
fake_process_instance.terminate = MagicMock()
fake_process_class = mocker.MagicMock()
fake_process_instance = mocker.MagicMock()
fake_process_instance.start = mocker.MagicMock()
fake_process_instance.join = mocker.MagicMock()
fake_process_instance.is_alive = mocker.MagicMock(return_value=False)
fake_process_instance.terminate = mocker.MagicMock()
fake_process_class.return_value = fake_process_instance
# Mock get_context to return a context with Queue that returns _FakeQueue
fake_ctx = MagicMock()
fake_ctx = mocker.MagicMock()
fake_ctx.Queue = lambda maxsize=0: _FakeQueue(maxsize=maxsize)
fake_ctx.Process = fake_process_class
@@ -365,7 +373,7 @@ def _setup_multiprocessing_mocks(monkeypatch):
)
def _setup_ipc_mocks(monkeypatch):
def _setup_ipc_mocks(monkeypatch: pytest.MonkeyPatch):
"""Helper function to set up IPC function mocks."""
# Mock _encode: simple serialization
@@ -385,7 +393,7 @@ def _setup_ipc_mocks(monkeypatch):
monkeypatch.setattr("vllm_omni.entrypoints.omni._set", _fake_set, raising=False)
def _setup_log_mocks(monkeypatch):
def _setup_log_mocks(monkeypatch: pytest.MonkeyPatch):
"""Helper function to set up logging and stats mocks."""
# Mock OrchestratorMetrics to be a simple class that doesn't require file operations
@@ -416,7 +424,7 @@ def _setup_log_mocks(monkeypatch):
)
def _setup_connector_mocks(monkeypatch, omni_module=None):
def _setup_connector_mocks(monkeypatch, mocker, omni_module=None):
"""Helper function to set up connector mocks for stage-to-stage forwarding.
If omni_module is provided, mocks directly on the module. Otherwise, uses string path.
@@ -429,7 +437,7 @@ def _setup_connector_mocks(monkeypatch, omni_module=None):
fake_connectors = {}
# Add connectors for common edges (0->1, 1->2, etc.)
for i in range(10): # Support up to 10 stages
fake_connectors[(str(i), str(i + 1))] = MagicMock()
fake_connectors[(str(i), str(i + 1))] = mocker.MagicMock()
return None, fake_connectors
if omni_module is not None:
@@ -476,15 +484,15 @@ def _setup_connector_adapter_mock(monkeypatch, omni_module):
@pytest.fixture(autouse=True)
def mock_get_config(monkeypatch):
def mock_get_config(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Auto-mock get_config and related model loading functions to avoid model path validation."""
# CRITICAL: Mock tokenizer-related imports FIRST, before any module imports
# This prevents ImportError when async_omni is imported (which happens via omni_stage)
import sys
fake_tokenizer = MagicMock()
fake_tokenizer.encode = MagicMock(return_value=[1, 2, 3])
fake_tokenizer.decode = MagicMock(return_value="test")
fake_tokenizer = mocker.MagicMock()
fake_tokenizer.encode = mocker.MagicMock(return_value=[1, 2, 3])
fake_tokenizer.decode = mocker.MagicMock(return_value="test")
# Mock init_tokenizer_from_configs (used in async_omni)
def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
@@ -555,7 +563,7 @@ def mock_get_config(monkeypatch):
setattr(async_omni_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
# Now mock get_config and other functions
fake_hf_config = MagicMock()
fake_hf_config = mocker.MagicMock()
fake_hf_config.model_type = "qwen2_5_omni"
def _mock_get_config(model, **kwargs):
@@ -597,7 +605,9 @@ def mock_get_config(monkeypatch):
)
def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_config):
def test_initialize_stage_configs_called_when_none(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that stage configs are auto-loaded when stage_configs_path is None."""
def _fake_loader(
@@ -627,11 +637,11 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
del sys.modules[module_name]
# Set up mocks
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
# Mock load_and_resolve_stage_configs
monkeypatch.setattr(
@@ -643,7 +653,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
# Replace OmniStage
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
@@ -651,7 +661,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
import vllm_omni.entrypoints.omni as omni_module
# Patch the imported class in the module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
from vllm_omni.entrypoints.omni import Omni
@@ -672,7 +682,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
assert len(omni._stages_ready) == 2
def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that generate raises ValueError when sampling_params_list length doesn't match."""
def _fake_loader(
@@ -693,11 +703,11 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -706,13 +716,13 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
from vllm_omni.entrypoints.omni import Omni
@@ -722,7 +732,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
omni.generate(prompts=["hi"], sampling_params_list=[])
def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test multi-stage generation pipeline with queue polling."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -750,8 +760,8 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
@@ -762,16 +772,16 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -828,7 +838,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
assert omni.stage_list[1].process_engine_inputs([], []) is not None
def test_generate_pipeline_with_batch_input(monkeypatch, fake_stage_config):
def test_generate_pipeline_with_batch_input(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test single-stage generation pipeline with multiple inputs in one batch."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -856,8 +866,8 @@ def test_generate_pipeline_with_batch_input(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
@@ -868,16 +878,16 @@ def test_generate_pipeline_with_batch_input(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -946,7 +956,9 @@ def test_generate_pipeline_with_batch_input(monkeypatch, fake_stage_config):
assert len(outputs) == 2
def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
def test_generate_no_final_output_returns_empty(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that generate returns empty list when all stages have final_output=False."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -975,8 +987,8 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
@@ -987,16 +999,16 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -1037,7 +1049,9 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
assert outputs == []
def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_config):
def test_generate_sampling_params_none_use_default(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that generate uses default sampling params when sampling_params_list is None."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -1066,8 +1080,8 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
@@ -1078,16 +1092,16 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -1121,7 +1135,7 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
omni.generate(prompts=["p"], sampling_params_list=None)
def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that _wait_for_stages_ready handles timeout correctly."""
def _fake_loader(
@@ -1142,11 +1156,11 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1158,21 +1172,21 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
class _FakeStageNoReady(_FakeStage):
def init_stage_worker(self, *args, **kwargs):
# Don't send stage_ready message
self._proc = MagicMock()
self._proc.start = MagicMock()
self._proc.join = MagicMock()
self._proc.is_alive = MagicMock(return_value=False)
self._proc.terminate = MagicMock()
self._proc = self._mocker.MagicMock()
self._proc.start = self._mocker.MagicMock()
self._proc.join = self._mocker.MagicMock()
self._proc.is_alive = self._mocker.MagicMock(return_value=False)
self._proc.terminate = self._mocker.MagicMock()
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStageNoReady(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStageNoReady(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
from vllm_omni.entrypoints.omni import Omni
@@ -1183,7 +1197,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
assert len(omni._stages_ready) == 0
def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that generate handles error messages from stages correctly."""
def _fake_loader(
@@ -1204,11 +1218,11 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1217,13 +1231,13 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -1264,7 +1278,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
assert len(outputs) == 1
def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that close() sends shutdown signal to all input queues."""
def _fake_loader(
@@ -1285,11 +1299,11 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1298,14 +1312,14 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
from vllm_omni.entrypoints.omni import Omni
+98 -90
View File
@@ -3,9 +3,9 @@ import uuid
import warnings
from queue import Empty, Queue
from typing import Any
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm import SamplingParams
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
@@ -124,7 +124,7 @@ class _FakeZmqQueue:
class _FakeStage:
"""Lightweight Stage stub for multi-process pipeline version with queue support."""
def __init__(self, config, stage_init_timeout: int = 300):
def __init__(self, mocker: MockerFixture, config, stage_init_timeout: int = 300):
# Handle both dict and object configs
if isinstance(config, dict):
config = _FakeStageConfig(config)
@@ -150,6 +150,8 @@ class _FakeStage:
self._out_q = None
self._proc = None # Mock process reference
self._stage_init_timeout = max(0, int(stage_init_timeout))
# NOTE: mocker fixture object **must be provided**
self._mocker = mocker
def attach_queues(self, in_q, out_q):
"""Attach input and output queues."""
@@ -168,11 +170,11 @@ class _FakeStage:
):
"""Mock init_stage_worker: don't start real process, just send stage_ready message."""
# Create a mock process object
self._proc = MagicMock()
self._proc.start = MagicMock()
self._proc.join = MagicMock()
self._proc.is_alive = MagicMock(return_value=False)
self._proc.terminate = MagicMock()
self._proc = self._mocker.MagicMock()
self._proc.start = self._mocker.MagicMock()
self._proc.join = self._mocker.MagicMock()
self._proc.is_alive = self._mocker.MagicMock(return_value=False)
self._proc.terminate = self._mocker.MagicMock()
# Send stage_ready message to output queue
if self._out_q is not None:
try:
@@ -236,21 +238,21 @@ def fake_stage_config():
}
def _setup_engine_mocks(monkeypatch):
def _setup_engine_mocks(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Helper function to set up common engine mocks."""
fake_engine = MagicMock()
fake_engine = mocker.MagicMock()
# Add necessary attributes to fake_engine
fake_engine.tokenizer = MagicMock()
fake_engine.tokenizer = mocker.MagicMock()
fake_engine.log_stats = False
fake_engine.vllm_config = MagicMock()
fake_engine.vllm_config.model_config = MagicMock()
fake_engine.vllm_config = mocker.MagicMock()
fake_engine.vllm_config.model_config = mocker.MagicMock()
fake_engine.vllm_config.model_config.io_processor_plugin = None
fake_engine.get_supported_tasks = MagicMock(return_value=[])
fake_engine.model_config = MagicMock()
fake_engine.get_supported_tasks = mocker.MagicMock(return_value=[])
fake_engine.model_config = mocker.MagicMock()
fake_engine.model_config.io_processor_plugin = None
# Add registry with resolve_model_cls method
fake_registry = MagicMock()
fake_registry.resolve_model_cls = MagicMock(return_value=(MagicMock(), "test_arch"))
fake_registry = mocker.MagicMock()
fake_registry.resolve_model_cls = mocker.MagicMock(return_value=(mocker.MagicMock(), "test_arch"))
fake_engine.model_config.registry = fake_registry
fake_engine.vllm_config.model_config.registry = fake_registry
@@ -299,21 +301,21 @@ def _setup_engine_mocks(monkeypatch):
)
def _setup_multiprocessing_mocks(monkeypatch):
def _setup_multiprocessing_mocks(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Helper function to set up multiprocessing mocks."""
import multiprocessing as mp
# Mock Process
fake_process_class = MagicMock()
fake_process_instance = MagicMock()
fake_process_instance.start = MagicMock()
fake_process_instance.join = MagicMock()
fake_process_instance.is_alive = MagicMock(return_value=False)
fake_process_instance.terminate = MagicMock()
fake_process_class = mocker.MagicMock()
fake_process_instance = mocker.MagicMock()
fake_process_instance.start = mocker.MagicMock()
fake_process_instance.join = mocker.MagicMock()
fake_process_instance.is_alive = mocker.MagicMock(return_value=False)
fake_process_instance.terminate = mocker.MagicMock()
fake_process_class.return_value = fake_process_instance
# Mock get_context to return a context with Queue that returns _FakeQueue
fake_ctx = MagicMock()
fake_ctx = mocker.MagicMock()
fake_ctx.Queue = lambda maxsize=0: _FakeQueue(maxsize=maxsize)
fake_ctx.Process = fake_process_class
@@ -337,7 +339,7 @@ def _setup_multiprocessing_mocks(monkeypatch):
)
def _setup_ipc_mocks(monkeypatch):
def _setup_ipc_mocks(monkeypatch: pytest.MonkeyPatch):
"""Helper function to set up IPC function mocks."""
# Mock _encode: simple serialization
@@ -357,7 +359,7 @@ def _setup_ipc_mocks(monkeypatch):
monkeypatch.setattr("vllm_omni.entrypoints.omni._set", _fake_set, raising=False)
def _setup_log_mocks(monkeypatch):
def _setup_log_mocks(monkeypatch: pytest.MonkeyPatch):
"""Helper function to set up logging and stats mocks."""
# Mock OrchestratorMetrics to be a simple class that doesn't require file operations
@@ -388,7 +390,7 @@ def _setup_log_mocks(monkeypatch):
)
def _setup_connector_mocks(monkeypatch, omni_module=None):
def _setup_connector_mocks(monkeypatch, mocker, omni_module=None):
"""Helper function to set up connector mocks for stage-to-stage forwarding.
If omni_module is provided, mocks directly on the module. Otherwise, uses string path.
@@ -401,7 +403,7 @@ def _setup_connector_mocks(monkeypatch, omni_module=None):
fake_connectors = {}
# Add connectors for common edges (0->1, 1->2, etc.)
for i in range(10): # Support up to 10 stages
fake_connectors[(str(i), str(i + 1))] = MagicMock()
fake_connectors[(str(i), str(i + 1))] = mocker.MagicMock()
return None, fake_connectors
if omni_module is not None:
@@ -448,15 +450,15 @@ def _setup_connector_adapter_mock(monkeypatch, omni_module):
@pytest.fixture(autouse=True)
def mock_get_config(monkeypatch):
def mock_get_config(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""Auto-mock get_config and related model loading functions to avoid model path validation."""
# CRITICAL: Mock tokenizer-related imports FIRST, before any module imports
# This prevents ImportError when async_omni is imported (which happens via omni_stage)
import sys
fake_tokenizer = MagicMock()
fake_tokenizer.encode = MagicMock(return_value=[1, 2, 3])
fake_tokenizer.decode = MagicMock(return_value="test")
fake_tokenizer = mocker.MagicMock()
fake_tokenizer.encode = mocker.MagicMock(return_value=[1, 2, 3])
fake_tokenizer.decode = mocker.MagicMock(return_value="test")
# Mock init_tokenizer_from_configs (used in async_omni)
def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
@@ -527,7 +529,7 @@ def mock_get_config(monkeypatch):
setattr(async_omni_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
# Now mock get_config and other functions
fake_hf_config = MagicMock()
fake_hf_config = mocker.MagicMock()
fake_hf_config.model_type = "qwen2_5_omni"
def _mock_get_config(model, **kwargs):
@@ -569,7 +571,9 @@ def mock_get_config(monkeypatch):
)
def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_config):
def test_initialize_stage_configs_called_when_none(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that stage configs are auto-loaded when stage_configs_path is None."""
def _fake_loader(
@@ -599,11 +603,11 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
del sys.modules[module_name]
# Set up mocks
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
# Mock load_and_resolve_stage_configs
monkeypatch.setattr(
@@ -615,7 +619,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
# Replace OmniStage
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
@@ -624,9 +628,9 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
# Patch the imported function and class in the module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
from vllm_omni.entrypoints.omni import Omni
@@ -646,7 +650,7 @@ def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_confi
assert len(omni._stages_ready) == 2
def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that generate raises ValueError when sampling_params_list length doesn't match."""
def _fake_loader(
@@ -669,11 +673,11 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -682,14 +686,14 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
from vllm_omni.entrypoints.omni import Omni
@@ -698,7 +702,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config):
omni.generate(prompts=["hi"], sampling_params_list=[])
def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test multi-stage generation pipeline with queue polling."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -724,11 +728,11 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -737,16 +741,16 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -803,7 +807,9 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
assert omni.stage_list[1].process_engine_inputs([], []) is not None
def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
def test_generate_no_final_output_returns_empty(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that generate returns empty list when all stages have final_output=False."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -830,11 +836,11 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -843,16 +849,16 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -893,7 +899,9 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
assert outputs == []
def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_config):
def test_generate_sampling_params_none_use_default(
monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config
):
"""Test that generate uses default sampling params when sampling_params_list is None."""
stage_cfg0 = dict(fake_stage_config)
stage_cfg0["stage_id"] = 0
@@ -920,11 +928,11 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -933,16 +941,16 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
# Apply connector and adapter mocks after importing omni module
_setup_connector_mocks(monkeypatch, omni_module)
_setup_connector_mocks(monkeypatch, mocker, omni_module)
_setup_connector_adapter_mock(monkeypatch, omni_module)
# Mock uuid.uuid4() to return a predictable value for request ID generation
@@ -976,7 +984,7 @@ def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_confi
omni.generate(prompts=["p"], sampling_params_list=None)
def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that _wait_for_stages_ready handles timeout correctly."""
def _fake_loader(
@@ -999,11 +1007,11 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1015,22 +1023,22 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
class _FakeStageNoReady(_FakeStage):
def init_stage_worker(self, *args, **kwargs):
# Don't send stage_ready message
self._proc = MagicMock()
self._proc.start = MagicMock()
self._proc.join = MagicMock()
self._proc.is_alive = MagicMock(return_value=False)
self._proc.terminate = MagicMock()
self._proc = self._mocker.MagicMock()
self._proc.start = self._mocker.MagicMock()
self._proc.join = self._mocker.MagicMock()
self._proc.is_alive = self._mocker.MagicMock(return_value=False)
self._proc.terminate = self._mocker.MagicMock()
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStageNoReady(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStageNoReady(mocker, cfg, **kwargs))
from vllm_omni.entrypoints.omni import Omni
@@ -1040,7 +1048,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config):
assert len(omni._stages_ready) == 0
def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that generate handles error messages from stages correctly."""
def _fake_loader(
@@ -1063,11 +1071,11 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1076,14 +1084,14 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
# Mock uuid.uuid4() to return a predictable value for request ID generation
test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000")
@@ -1123,7 +1131,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
assert len(outputs) == 1
def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, fake_stage_config):
"""Test that close() sends shutdown signal to all input queues."""
def _fake_loader(
@@ -1146,11 +1154,11 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
if module_name in sys.modules:
del sys.modules[module_name]
_setup_engine_mocks(monkeypatch)
_setup_multiprocessing_mocks(monkeypatch)
_setup_engine_mocks(monkeypatch, mocker)
_setup_multiprocessing_mocks(monkeypatch, mocker)
_setup_ipc_mocks(monkeypatch)
_setup_log_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch)
_setup_connector_mocks(monkeypatch, mocker)
monkeypatch.setattr(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
@@ -1159,14 +1167,14 @@ def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config):
)
monkeypatch.setattr(
"vllm_omni.entrypoints.omni_stage.OmniStage",
lambda cfg, **kwargs: _FakeStage(cfg, **kwargs),
lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs),
raising=False,
)
import vllm_omni.entrypoints.omni as omni_module
monkeypatch.setattr(omni_module, "load_and_resolve_stage_configs", _fake_loader)
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs))
monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(mocker, cfg, **kwargs))
from vllm_omni.entrypoints.omni import Omni
+10 -8
View File
@@ -1,8 +1,8 @@
import os
import sys
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.entrypoints.stage_utils import set_stage_devices
@@ -45,16 +45,18 @@ def _make_dummy_torch(call_log):
return _Torch
def _make_mock_platform(device_type: str = "cuda", env_var: str = "CUDA_VISIBLE_DEVICES"):
"""Create a mock platform for testing."""
mock_platform = MagicMock()
def _make_mock_platform(mocker, device_type: str = "cuda", env_var: str = "CUDA_VISIBLE_DEVICES"):
"""Create a mock platform for testing.
mocker object has to be passed in to utilize this helper function.
"""
mock_platform = mocker.MagicMock()
mock_platform.device_type = device_type
mock_platform.device_control_env_var = env_var
return mock_platform
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_set_stage_devices_respects_logical_ids(monkeypatch):
def test_set_stage_devices_respects_logical_ids(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
call_log: list[int] = []
@@ -62,7 +64,7 @@ def test_set_stage_devices_respects_logical_ids(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", dummy_torch)
# Mock the platform at the source module where it's defined
mock_platform = _make_mock_platform(device_type="cuda", env_var="CUDA_VISIBLE_DEVICES")
mock_platform = _make_mock_platform(mocker, device_type="cuda", env_var="CUDA_VISIBLE_DEVICES")
monkeypatch.setattr(
"vllm_omni.platforms.current_omni_platform",
mock_platform,
@@ -74,7 +76,7 @@ def test_set_stage_devices_respects_logical_ids(monkeypatch):
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
def test_set_stage_devices_npu_platform(monkeypatch):
def test_set_stage_devices_npu_platform(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
"""Test that set_stage_devices works correctly for NPU platform."""
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "4,5")
call_log: list[int] = []
@@ -99,7 +101,7 @@ def test_set_stage_devices_npu_platform(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _NpuTorch)
# Mock NPU platform at the source module where it's defined
mock_platform = _make_mock_platform(device_type="npu", env_var="ASCEND_RT_VISIBLE_DEVICES")
mock_platform = _make_mock_platform(mocker, device_type="npu", env_var="ASCEND_RT_VISIBLE_DEVICES")
monkeypatch.setattr(
"vllm_omni.platforms.current_omni_platform",
mock_platform,
+22 -22
View File
@@ -2,9 +2,9 @@
from collections import Counter
from dataclasses import dataclass
from unittest.mock import patch
import pytest
from pytest_mock import MockerFixture
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.engine.arg_utils import OmniEngineArgs
@@ -140,7 +140,7 @@ class TestFilterDictLikeObject:
assert 42 in result
assert (1, 2) in result
def test_dict_with_recursive_structure(self):
def test_dict_with_recursive_structure(self, mocker: MockerFixture):
"""Test filtering dict with recursive/complex nested structure."""
input_dict = {
"level1": {
@@ -152,15 +152,15 @@ class TestFilterDictLikeObject:
"normal": "value",
}
with patch("vllm_omni.entrypoints.utils.logger"):
result = _filter_dict_like_object(input_dict)
mocker.patch("vllm_omni.entrypoints.utils.logger")
result = _filter_dict_like_object(input_dict)
# Normal key should exist
assert "normal" in result
# Level1 should exist
assert "level1" in result
# Normal key should exist
assert "normal" in result
# Level1 should exist
assert "level1" in result
def test_integration_with_convert_dataclasses(self):
def test_integration_with_convert_dataclasses(self, mocker: MockerFixture):
"""Test that _filter_dict_like_object integrates properly with _convert_dataclasses_to_dict."""
@dataclass
@@ -174,32 +174,32 @@ class TestFilterDictLikeObject:
"normal": "value",
}
with patch("vllm_omni.entrypoints.utils.logger"):
result = _filter_dict_like_object(input_dict)
mocker.patch("vllm_omni.entrypoints.utils.logger")
result = _filter_dict_like_object(input_dict)
# Callable should be filtered
assert "func" not in result
# Config should be converted to dict
assert "config" in result
assert "normal" in result
# Callable should be filtered
assert "func" not in result
# Config should be converted to dict
assert "config" in result
assert "normal" in result
class TestConvertDataclassesToDict:
"""Test suite for _convert_dataclasses_to_dict function."""
def test_uses_filter_dict_like_object(self):
def test_uses_filter_dict_like_object(self, mocker: MockerFixture):
"""Test that _convert_dataclasses_to_dict uses _filter_dict_like_object for dicts."""
input_dict = {
"normal": "value",
"callable": lambda x: x,
}
with patch("vllm_omni.entrypoints.utils.logger"):
result = _convert_dataclasses_to_dict(input_dict)
mocker.patch("vllm_omni.entrypoints.utils.logger")
result = _convert_dataclasses_to_dict(input_dict)
# Callable should be filtered out by _filter_dict_like_object
assert "normal" in result
assert "callable" not in result
# Callable should be filtered out by _filter_dict_like_object
assert "normal" in result
assert "callable" not in result
class TestFilterDataclassKwargs:
@@ -236,7 +236,6 @@ def test_modality_control_003(omni_server) -> None:
# TODO: Verify the E2E latency after confirmation baseline.
@pytest.mark.skip(reason="There is a known issue with stream error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@@ -0,0 +1,318 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
correctly assigned to audio/image/video token positions.
Regression test for: https://github.com/vllm-project/vllm/issues/34506
- Non-interleaved mixed modalities (audio + image + video) should correctly
assign audio embeddings to audio positions, image to image, video to video.
- Interleaved (use_audio_in_video) should also work correctly.
"""
from unittest.mock import Mock
import pytest
import torch
from vllm.model_executor.models.qwen2_5_omni_thinker import (
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
# Fake token IDs
AUDIO_TOKEN_ID = 1001
IMAGE_TOKEN_ID = 1002
VIDEO_TOKEN_ID = 1003
TEXT_TOKEN_ID = 0
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_token_seq(audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2):
"""
Build a flat token sequence:
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
[text_sep] [VIDEO * video_n] [text_sep]
Returns (input_ids tensor, is_multimodal mask, positions dict).
"""
tokens = (
[TEXT_TOKEN_ID] * text_prefix
+ [AUDIO_TOKEN_ID] * audio_n
+ [TEXT_TOKEN_ID] * text_sep
+ [IMAGE_TOKEN_ID] * image_n
+ [TEXT_TOKEN_ID] * text_sep
+ [VIDEO_TOKEN_ID] * video_n
+ [TEXT_TOKEN_ID] * text_sep
)
input_ids = torch.tensor(tokens)
is_multimodal = (input_ids == AUDIO_TOKEN_ID) | (input_ids == IMAGE_TOKEN_ID) | (input_ids == VIDEO_TOKEN_ID)
return input_ids, is_multimodal
def make_interleaved_seq(video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2):
"""
Build an interleaved sequence like use_audio_in_video:
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
"""
tokens = [TEXT_TOKEN_ID] * text_prefix
for v, a in zip(video_chunks, audio_chunks):
tokens += [VIDEO_TOKEN_ID] * v + [AUDIO_TOKEN_ID] * a
input_ids = torch.tensor(tokens)
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
return input_ids, is_multimodal
# ---------------------------------------------------------------------------
# Tests for check_interleaved_audio_video
# ---------------------------------------------------------------------------
class TestCheckInterleavedAudioVideo:
def test_non_interleaved_audio_then_video(self):
"""Audio entirely before video -> not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 0, 4)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(is_video, is_audio, is_video.sum().item(), is_audio.sum().item())
def test_non_interleaved_with_image(self):
"""Audio + image + video (the mixed_modalities case) -> not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 4, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(is_video, is_audio, is_video.sum().item(), is_audio.sum().item())
def test_no_audio(self):
"""Video only -> not interleaved."""
input_ids, is_multimodal = make_token_seq(0, 0, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(is_video, is_audio, is_video.sum().item(), is_audio.sum().item())
def test_interleaved(self):
"""V A V A interleaved -> True."""
input_ids, is_multimodal = make_interleaved_seq([4, 4], [3, 3])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(is_video, is_audio, is_video.sum().item(), is_audio.sum().item())
# ---------------------------------------------------------------------------
# Tests for embed_input_ids via a minimal mock
# ---------------------------------------------------------------------------
def make_mock_model(hidden: int = 8):
"""
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
that has enough structure to run embed_input_ids.
"""
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerForConditionalGeneration,
)
model = Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
# Config with token IDs
cfg = Mock()
cfg.video_token_index = VIDEO_TOKEN_ID
cfg.audio_token_index = AUDIO_TOKEN_ID
model.config = cfg
def fake_lm_embed(ids: torch.Tensor) -> torch.Tensor:
# Use .clone() so the tensor is contiguous (expand() creates a strided
# view with shared memory, which masked_scatter_ cannot handle).
return ids.float().unsqueeze(-1).expand(-1, hidden).clone()
lang_model = Mock()
lang_model.embed_input_ids = fake_lm_embed
model.get_language_model = Mock(return_value=lang_model)
from vllm.model_executor.models.interfaces import SupportsMultiModal
model._embed_text_input_ids = lambda *a, **kw: SupportsMultiModal._embed_text_input_ids(model, *a, **kw)
def fake_super_embed(ids, mm_embs=None, *, is_multimodal=None, handle_oov_mm_token=False):
return SupportsMultiModal.embed_input_ids(
model,
ids,
mm_embs,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
model.embed_input_ids = lambda *a, **kw: Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids(model, *a, **kw)
model._super_embed_input_ids = fake_super_embed
return model, hidden
def build_mm_embeds(audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0):
"""
Build multimodal_embeddings list in position order (audio, image, video).
Each embedding is filled with a distinct constant so we can verify placement.
"""
embs = []
if audio_n:
embs.append(torch.full((audio_n, hidden), audio_val))
if image_n:
embs.append(torch.full((image_n, hidden), image_val))
if video_n:
embs.append(torch.full((video_n, hidden), video_val))
return embs
class TestEmbedInputIds:
def _run(self, audio_n, image_n, video_n, hidden=8):
"""
Run embed_input_ids for a non-interleaved mixed-modality sequence.
Returns (result_embeds, input_ids, is_multimodal).
"""
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
mm_embeds = build_mm_embeds(audio_n, image_n, video_n, hidden)
model, _ = make_mock_model(hidden)
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
return result, input_ids, is_multimodal
def test_audio_only(self):
"""Audio-only: audio positions get audio embeddings."""
audio_n, hidden = 5, 8
audio_val = 10.0
result, input_ids, is_multimodal = self._run(audio_n, 0, 0, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Audio positions should get audio embeddings"
)
def test_video_only(self):
"""Video-only: video positions get video embeddings."""
video_n, hidden = 6, 8
video_val = 30.0
result, input_ids, is_multimodal = self._run(0, 0, video_n, hidden)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Video positions should get video embeddings"
)
def test_mixed_modalities_audio_goes_to_audio_pos(self):
"""
Regression test for GitHub issue #34506:
With audio + image + video (non-interleaved), audio positions must
receive audio embeddings (not image or video embeddings).
"""
audio_n, image_n, video_n, hidden = 5, 4, 6, 8
audio_val, image_val, video_val = 10.0, 20.0, 30.0
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
image_pos = (input_ids == IMAGE_TOKEN_ID).nonzero(as_tuple=True)[0]
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
mean_a = result[audio_pos].mean().item()
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
f"Audio emb wrong: expected {audio_val}, got mean={mean_a:.1f}"
)
mean_i = result[image_pos].mean().item()
assert result[image_pos].allclose(torch.full((image_n, hidden), image_val)), (
f"Image emb wrong: expected {image_val}, got mean={mean_i:.1f}"
)
mean_v = result[video_pos].mean().item()
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
f"Video emb wrong: expected {video_val}, got mean={mean_v:.1f}"
)
def test_text_positions_unchanged(self):
"""Text positions should keep their text embeddings."""
audio_n, image_n, video_n, hidden = 3, 2, 4, 8
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
text_pos = (~is_multimodal).nonzero(as_tuple=True)[0]
# Text tokens have value TEXT_TOKEN_ID=0, so embed -> 0.0
assert result[text_pos].allclose(torch.zeros(len(text_pos), hidden)), (
"Text positions should keep text embeddings"
)
def test_interleaved_use_audio_in_video(self):
"""
Interleaved (use_audio_in_video): video chunks interleaved with audio.
Video embeddings must go to video positions, audio to audio positions.
"""
hidden = 8
audio_val, video_val = 10.0, 30.0
video_chunks = [4, 4]
audio_chunks = [3, 3]
input_ids, is_multimodal = make_interleaved_seq(video_chunks, audio_chunks)
video_n = sum(video_chunks) # 8
audio_n = sum(audio_chunks) # 6
mm_embeds = [
torch.full((video_n, hidden), video_val),
torch.full((audio_n, hidden), audio_val),
]
model, _ = make_mock_model(hidden)
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Interleaved: video positions should get video embeddings"
)
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Interleaved: audio positions should get audio embeddings"
)
# ---------------------------------------------------------------------------
# Tests for merge_interleaved_embeddings helper
# ---------------------------------------------------------------------------
class TestMergeInterleavedEmbeddings:
def test_basic_interleaved(self):
"""Video chunks + audio chunks scattered to correct positions."""
hidden = 4
input_ids, is_multimodal = make_interleaved_seq([3, 3], [2, 2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
num_video = is_video.sum().item() # 6
num_audio = is_audio.sum().item() # 4
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
torch.full((num_video, hidden), 30.0),
torch.full((num_audio, hidden), 10.0),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
audio_pos = is_audio.nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
if __name__ == "__main__":
pytest.main([__file__, "-v"])
Whitespace-only changes.
@@ -0,0 +1,272 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for CUDA Graph decoder wrapper numerical equivalence.
Verifies that CUDA Graph-accelerated decoding produces results equivalent
to eager mode, with special attention to padding cases where zero-padding
may introduce small numerical differences due to attention and convolution.
Architecture note: the real Qwen3TTSTokenizerV2Decoder uses causal
convolutions, so zero-padding on the right has minimal impact (~2e-3).
The synthetic decoder here uses standard (non-causal) Conv1d for a
worst-case test of the wrapper mechanism.
"""
import importlib.util
import os
import pytest
import torch
import torch.nn as nn
pytestmark = [pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")]
DEVICE = torch.device("cuda:0")
NUM_QUANTIZERS = 8
TOTAL_UPSAMPLE = 4
# Load CUDAGraphDecoderWrapper: try package import first, fall back to direct file load
try:
from vllm_omni.model_executor.models.qwen3_tts.cuda_graph_decoder_wrapper import CUDAGraphDecoderWrapper
except Exception:
_WRAPPER_PATH = os.path.join(
os.path.dirname(__file__),
os.pardir,
os.pardir,
os.pardir,
os.pardir,
"vllm_omni",
"model_executor",
"models",
"qwen3_tts",
"cuda_graph_decoder_wrapper.py",
)
_spec = importlib.util.spec_from_file_location("cuda_graph_decoder_wrapper", os.path.abspath(_WRAPPER_PATH))
_mod = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_mod)
CUDAGraphDecoderWrapper = _mod.CUDAGraphDecoderWrapper
class SyntheticDecoder(nn.Module):
"""A small decoder mimicking Qwen3TTSTokenizerV2Decoder's interface.
Uses Conv1d layers so that zero-padding can affect neighboring positions
via the receptive field, providing a worst-case test for padding effects.
"""
def __init__(self, num_quantizers=NUM_QUANTIZERS, total_upsample=TOTAL_UPSAMPLE):
super().__init__()
hidden = 32
self.total_upsample = total_upsample
self.embed = nn.Conv1d(num_quantizers, hidden, kernel_size=3, padding=1)
self.conv1 = nn.Conv1d(hidden, hidden, kernel_size=5, padding=2)
self.conv2 = nn.Conv1d(hidden, hidden, kernel_size=3, padding=1)
self.upsample = nn.ConvTranspose1d(hidden, hidden, kernel_size=total_upsample, stride=total_upsample)
self.out = nn.Conv1d(hidden, 1, kernel_size=1)
def forward(self, codes):
x = codes.float()
x = torch.relu(self.embed(x))
x = torch.relu(self.conv1(x))
x = torch.relu(self.conv2(x))
x = self.upsample(x)
return self.out(x).clamp(min=-1, max=1)
@pytest.fixture(scope="module")
def decoder():
"""Create a synthetic decoder on CUDA with fixed weights."""
torch.manual_seed(42)
return SyntheticDecoder().to(DEVICE).eval()
@pytest.fixture(scope="module")
def wrapper(decoder):
"""Create a warmed-up CUDAGraphDecoderWrapper."""
w = CUDAGraphDecoderWrapper(
decoder=decoder,
capture_sizes=[25, 50, 100],
num_quantizers=NUM_QUANTIZERS,
enabled=True,
)
w.warmup(DEVICE)
return w
def _random_codes(seq_len, device=DEVICE):
return torch.randint(0, 100, (1, NUM_QUANTIZERS, seq_len), dtype=torch.long, device=device)
# ──────────────────────────────────────────────────────────────────
# 1. Exact-size inputs (no padding needed) → bit-identical
# ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("seq_len", [25, 50, 100])
def test_exact_size_numerical_equivalence(decoder, wrapper, seq_len):
"""When input exactly matches a capture size, output must be bit-identical."""
codes = _random_codes(seq_len)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
# ──────────────────────────────────────────────────────────────────
# 2. Padded inputs (zero-padding to nearest capture size)
# ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_output_shape_and_length(decoder, wrapper, seq_len):
"""Padded decode must return output trimmed to actual input length."""
codes = _random_codes(seq_len)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
expected_len = seq_len * TOTAL_UPSAMPLE
assert graph_out.shape == eager_out.shape
assert graph_out.shape[-1] == expected_len
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_interior_positions_close(decoder, wrapper, seq_len):
"""Interior positions (away from padding boundary) should be very close.
The conv receptive field is at most 5 (kernel_size=5), so positions
more than 2 timesteps from the end (times the upsample factor) should
be nearly identical between eager and graph modes.
"""
codes = _random_codes(seq_len)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
# Exclude the last (receptive_field * upsample) positions from strict check
boundary = 3 * TOTAL_UPSAMPLE # conservative: 3 positions * 4x upsample
if eager_out.shape[-1] > boundary:
interior_eager = eager_out[..., :(-boundary)]
interior_graph = graph_out[..., :(-boundary)]
torch.testing.assert_close(interior_graph, interior_eager, atol=1e-5, rtol=1e-5)
@pytest.mark.parametrize("seq_len", [10, 30, 47, 73, 99])
def test_padded_output_bounded(decoder, wrapper, seq_len):
"""Padded output values must remain in [-1, 1] and max diff should be bounded."""
codes = _random_codes(seq_len)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
assert graph_out.min() >= -1.0 and graph_out.max() <= 1.0
max_diff = (graph_out - eager_out).abs().max().item()
# With non-causal conv, boundary diffs can be large (~0.5).
# The real causal decoder shows ~2e-3.
assert max_diff < 1.0, f"Max diff {max_diff} exceeds bound"
# ──────────────────────────────────────────────────────────────────
# 3. Fallback to eager (size exceeds all capture sizes) → bit-identical
# ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("seq_len", [101, 150, 200])
def test_fallback_eager_exact_match(decoder, wrapper, seq_len):
"""Input larger than all capture sizes falls back to eager -> bit-identical."""
codes = _random_codes(seq_len)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
# ──────────────────────────────────────────────────────────────────
# 4. Chunked decode equivalence
# ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("total_len", [60, 100, 150, 250])
def test_chunked_decode_shape_match(decoder, wrapper, total_len):
"""Chunked decode output shape must match between eager and graph modes."""
codes = _random_codes(total_len)
chunk_size, ctx = 50, 10
with torch.no_grad():
eager_out = _eager_chunked(decoder, codes, chunk_size, ctx)
graph_out = wrapper.chunked_decode_with_cudagraph(codes, chunk_size=chunk_size, left_context_size=ctx)
assert eager_out.shape == graph_out.shape
@pytest.mark.parametrize("total_len", [50, 100])
def test_chunked_decode_exact_size_equivalence(decoder, wrapper, total_len):
"""Chunked decode with chunks matching capture sizes should be bit-identical."""
codes = _random_codes(total_len)
# chunk_size=50 matches a capture size exactly, no context overlap
chunk_size, ctx = 50, 0
with torch.no_grad():
eager_out = _eager_chunked(decoder, codes, chunk_size, ctx)
graph_out = wrapper.chunked_decode_with_cudagraph(codes, chunk_size=chunk_size, left_context_size=ctx)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
def _eager_chunked(decoder, codes, chunk_size, left_context_size):
"""Eager chunked decode matching the real decoder's chunked_decode logic."""
wavs = []
start = 0
total_len = codes.shape[-1]
while start < total_len:
end = min(start + chunk_size, total_len)
ctx = left_context_size if start - left_context_size > 0 else start
chunk = codes[..., start - ctx : end]
wav = decoder(chunk)
wavs.append(wav[..., ctx * decoder.total_upsample :])
start = end
return torch.cat(wavs, dim=-1)
# ──────────────────────────────────────────────────────────────────
# 5. Edge cases and control tests
# ──────────────────────────────────────────────────────────────────
def test_single_frame(decoder, wrapper):
"""Single-frame input (seq_len=1) should work with padding."""
codes = _random_codes(1)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
assert graph_out.shape == eager_out.shape
assert graph_out.shape[-1] == TOTAL_UPSAMPLE
def test_disabled_wrapper_matches_eager(decoder, wrapper):
"""Disabled wrapper should produce bit-identical output to eager."""
codes = _random_codes(30)
wrapper.enabled = False
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
wrapper.enabled = True
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
def test_batch_size_gt1_falls_back(decoder, wrapper):
"""Batch size > 1 should fall back to eager (bit-identical)."""
codes = torch.randint(0, 100, (2, NUM_QUANTIZERS, 25), dtype=torch.long, device=DEVICE)
with torch.no_grad():
eager_out = decoder(codes)
graph_out = wrapper.decode(codes)
torch.testing.assert_close(graph_out, eager_out, atol=0, rtol=0)
def test_deterministic_across_calls(decoder, wrapper):
"""Same input should produce identical CUDA graph output across calls."""
codes = _random_codes(30)
with torch.no_grad():
out1 = wrapper.decode(codes)
out2 = wrapper.decode(codes)
torch.testing.assert_close(out1, out2, atol=0, rtol=0)
@@ -0,0 +1,72 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from types import SimpleNamespace
import torch
from vllm_omni.model_executor.stage_input_processors.qwen3_tts import talker2code2wav_async_chunk
def _req(external_req_id: str, *, finished: bool):
return SimpleNamespace(
external_req_id=external_req_id,
is_finished=lambda: finished,
)
def test_talker2code2wav_async_chunk_does_not_emit_empty_chunk_when_not_finished():
transfer_manager = SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
connector=SimpleNamespace(config={"extra": {"codec_chunk_frames": 25, "codec_left_context_frames": 25}}),
)
request = _req("rid-empty", finished=False)
payload = talker2code2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output={"audio_codes": torch.zeros((0,))},
request=request,
)
assert payload is None
def test_talker2code2wav_async_chunk_flushes_tail_when_finished_without_pooler_output():
transfer_manager = SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
connector=SimpleNamespace(config={"extra": {"codec_chunk_frames": 25, "codec_left_context_frames": 25}}),
)
request_id = "rid-tail"
transfer_manager.code_prompt_token_ids[request_id] = [[1, 2, 3, 4] for _ in range(24)]
request = _req(request_id, finished=True)
payload = talker2code2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None, # e.g. EOS step with no audio_codes
request=request,
)
assert payload is not None
assert payload["finished"].item() is True
# ctx_frames header + flat codes
assert len(payload["code_predictor_codes"]) == 1 + 4 * 24
def test_talker2code2wav_async_chunk_emits_eof_marker_when_finished_with_no_frames():
transfer_manager = SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
connector=SimpleNamespace(config={"extra": {"codec_chunk_frames": 25, "codec_left_context_frames": 25}}),
)
request = _req("rid-eof", finished=True)
payload = talker2code2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None,
request=request,
)
assert payload == {
"code_predictor_codes": [],
"finished": torch.tensor(True, dtype=torch.bool),
}
+23 -20
View File
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for OmniRequestOutput class."""
from unittest.mock import Mock
import pytest
from PIL import Image
from pytest_mock import MockerFixture
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestOmniRequestOutput:
"""Tests for OmniRequestOutput class."""
@@ -28,17 +30,17 @@ class TestOmniRequestOutput:
assert output.is_diffusion_output
assert output.num_images == 1
def test_from_pipeline(self):
def test_from_pipeline(self, mocker: MockerFixture):
"""Test creating output from pipeline stage."""
mock_request_output = Mock()
mock_request_output = mocker.Mock()
mock_request_output.request_id = "pipeline-123"
mock_request_output.prompt_token_ids = [1, 2, 3]
mock_request_output.outputs = [Mock()]
mock_request_output.outputs = [mocker.Mock()]
mock_request_output.encoder_prompt_token_ids = None
mock_request_output.prompt_logprobs = None
mock_request_output.num_cached_tokens = 10
mock_request_output.kv_transfer_params = None
mock_request_output.multimodal_output = {"image": Mock()}
mock_request_output.multimodal_output = {"image": mocker.Mock()}
output = OmniRequestOutput.from_pipeline(
stage_id=0,
@@ -51,9 +53,9 @@ class TestOmniRequestOutput:
assert output.final_output_type == "text"
assert output.is_pipeline_output
def test_prompt_token_ids_property(self):
def test_prompt_token_ids_property(self, mocker: MockerFixture):
"""Test prompt_token_ids property for streaming compatibility."""
mock_request_output = Mock()
mock_request_output = mocker.Mock()
mock_request_output.prompt_token_ids = [1, 2, 3, 4, 5]
output = OmniRequestOutput.from_pipeline(
@@ -73,10 +75,10 @@ class TestOmniRequestOutput:
)
assert output.prompt_token_ids is None
def test_outputs_property(self):
def test_outputs_property(self, mocker: MockerFixture):
"""Test outputs property for chat completion compatibility."""
mock_output = Mock()
mock_request_output = Mock()
mock_output = mocker.Mock()
mock_request_output = mocker.Mock()
mock_request_output.outputs = [mock_output]
output = OmniRequestOutput.from_pipeline(
@@ -96,9 +98,9 @@ class TestOmniRequestOutput:
)
assert output.outputs == []
def test_encoder_prompt_token_ids_property(self):
def test_encoder_prompt_token_ids_property(self, mocker: MockerFixture):
"""Test encoder_prompt_token_ids property."""
mock_request_output = Mock()
mock_request_output = mocker.Mock()
mock_request_output.encoder_prompt_token_ids = [10, 20, 30]
output = OmniRequestOutput.from_pipeline(
@@ -109,9 +111,9 @@ class TestOmniRequestOutput:
assert output.encoder_prompt_token_ids == [10, 20, 30]
def test_num_cached_tokens_property(self):
def test_num_cached_tokens_property(self, mocker: MockerFixture):
"""Test num_cached_tokens property."""
mock_request_output = Mock()
mock_request_output = mocker.Mock()
mock_request_output.num_cached_tokens = 42
output = OmniRequestOutput.from_pipeline(
@@ -122,11 +124,12 @@ class TestOmniRequestOutput:
assert output.num_cached_tokens == 42
def test_multimodal_output_property(self):
def test_multimodal_output_property(self, mocker: MockerFixture):
"""Test multimodal_output property."""
mock_request_output = Mock()
mock_audio = Mock()
mock_request_output = mocker.Mock()
mock_audio = mocker.Mock()
expected_output = {"audio": mock_audio}
mock_request_output.outputs = []
mock_request_output.multimodal_output = expected_output
output = OmniRequestOutput.from_pipeline(
@@ -153,9 +156,9 @@ class TestOmniRequestOutput:
assert result["num_images"] == 1
assert result["prompt"] == "a cat"
def test_to_dict_pipeline(self):
def test_to_dict_pipeline(self, mocker: MockerFixture):
"""Test to_dict for pipeline output."""
mock_request_output = Mock()
mock_request_output = mocker.Mock()
mock_request_output.request_id = "pipeline-123"
output = OmniRequestOutput.from_pipeline(
@@ -29,6 +29,15 @@ class DummyReqState:
pass
class MiMoAudioForConditionalGeneration(torch.nn.Module):
"""Dummy model whose class name must exactly match the production check."""
def __init__(self):
super().__init__()
# No real forward needed for these tests.
class DummyTalkerMTP(torch.nn.Module):
"""A fake talker_mtp module for deterministic CPU testing."""
@@ -82,6 +91,30 @@ def _make_runner(req_ids=("r1", "r2"), hidden_size=4):
return runner
def _make_runner_for_mimo(req_id="r_mimo"):
"""Create a minimal runner with MiMoAudio-like model and request state."""
runner = object.__new__(OmniGPUModelRunner)
runner.model = MiMoAudioForConditionalGeneration()
# Minimal vllm_config / model_config used by helper.
class _DummyModelConfig:
async_chunk = False
class _DummyVllmConfig:
model_config = _DummyModelConfig()
runner.vllm_config = _DummyVllmConfig()
# Attach a single request state with mm_features and additional_information_cpu.
req_state = DummyReqState()
req_state.mm_features = ["mm_feature_obj"]
req_state.additional_information_cpu = {"some_key": "some_value"}
runner.requests = {req_id: req_state}
return runner
def test_talker_mtp_forward_cpu_updates_inputs_and_info(monkeypatch):
# Patch the module-level `set_forward_context` symbol used inside
# OmniGPUModelRunner._talker_mtp_forward.
@@ -132,3 +165,32 @@ def test_talker_mtp_forward_cpu_empty_batch_noop(monkeypatch):
# Ensure no changes were made
assert torch.allclose(inputs_embeds, before)
def test_maybe_attach_mimo_audio_req_infos_enriches_dict():
runner = _make_runner_for_mimo()
req_id = "r_mimo"
req_state = runner.requests[req_id]
# Existing req_infos should be copied and enriched, not mutated in place.
original_req_infos = {"existing": 1}
enriched = OmniGPUModelRunner._maybe_attach_mimo_audio_req_infos(runner, req_state, original_req_infos, req_id)
assert enriched is not original_req_infos
assert enriched["existing"] == 1
# mm_features should be filled from req_state when missing
assert enriched["mm_features"] == req_state.mm_features
# req_id should always be attached
assert enriched["req_id"] == req_id
def test_maybe_attach_mimo_audio_req_infos_no_req_state_returns_input():
runner = _make_runner_for_mimo()
req_id = "missing"
req_state = None
req_infos = {"k": "v"}
result = OmniGPUModelRunner._maybe_attach_mimo_audio_req_infos(runner, req_state, req_infos, req_id)
# When no req_state, helper should be a no-op.
assert result is req_infos
+71 -77
View File
@@ -1,52 +1,54 @@
"""Tests for process-scoped GPU memory accounting."""
import os
from unittest import mock
import pytest
from pytest_mock import MockerFixture
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestParseCudaVisibleDevices:
def test_empty(self):
def test_empty(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import parse_cuda_visible_devices
with mock.patch.dict(os.environ, {}, clear=True):
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
assert parse_cuda_visible_devices() == []
mocker.patch.dict(os.environ, {}, clear=True)
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
assert parse_cuda_visible_devices() == []
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": ""}):
assert parse_cuda_visible_devices() == []
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": ""})
assert parse_cuda_visible_devices() == []
def test_integer_indices(self):
def test_integer_indices(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import parse_cuda_visible_devices
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2,3,5"}):
assert parse_cuda_visible_devices() == [2, 3, 5]
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2,3,5"})
assert parse_cuda_visible_devices() == [2, 3, 5]
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0"}):
assert parse_cuda_visible_devices() == [0]
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0"})
assert parse_cuda_visible_devices() == [0]
def test_uuids(self):
def test_uuids(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import parse_cuda_visible_devices
uuid1 = "GPU-12345678-1234-1234-1234-123456789abc"
uuid2 = "GPU-87654321-4321-4321-4321-cba987654321"
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": f"{uuid1},{uuid2}"}):
assert parse_cuda_visible_devices() == [uuid1, uuid2]
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": f"{uuid1},{uuid2}"})
assert parse_cuda_visible_devices() == [uuid1, uuid2]
def test_mig_ids(self):
def test_mig_ids(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import parse_cuda_visible_devices
mig1 = "MIG-GPU-12345678-1234-1234-1234-123456789abc/0/0"
mig2 = "MIG-GPU-12345678-1234-1234-1234-123456789abc/1/0"
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": f"{mig1},{mig2}"}):
assert parse_cuda_visible_devices() == [mig1, mig2]
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": f"{mig1},{mig2}"})
assert parse_cuda_visible_devices() == [mig1, mig2]
def test_spaces(self):
def test_spaces(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import parse_cuda_visible_devices
with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": " 2 , 3 , 5 "}):
assert parse_cuda_visible_devices() == [2, 3, 5]
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": " 2 , 3 , 5 "})
assert parse_cuda_visible_devices() == [2, 3, 5]
class TestGetProcessGpuMemory:
@@ -68,87 +70,79 @@ class TestGetProcessGpuMemory:
del tensor
torch.cuda.empty_cache()
def test_raises_on_invalid_device(self):
def test_raises_on_invalid_device(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
with (
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
mock.patch("vllm.third_party.pynvml.nvmlDeviceGetCount", return_value=1),
):
with pytest.raises(RuntimeError, match="Invalid GPU device"):
get_process_gpu_memory(5)
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
mocker.patch("vllm.third_party.pynvml.nvmlDeviceGetCount", return_value=1)
def test_returns_zero_when_process_not_found(self):
with pytest.raises(RuntimeError, match="Invalid GPU device"):
get_process_gpu_memory(5)
def test_returns_zero_when_process_not_found(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
with (
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
mock.patch("vllm.third_party.pynvml.nvmlDeviceGetCount", return_value=8),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetHandleByIndex"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetComputeRunningProcesses", return_value=[]),
):
memory = get_process_gpu_memory(0)
assert memory == 0
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
mocker.patch("vllm.third_party.pynvml.nvmlDeviceGetCount", return_value=8)
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetHandleByIndex")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetComputeRunningProcesses", return_value=[])
def test_uses_uuid_when_provided(self):
memory = get_process_gpu_memory(0)
assert memory == 0
def test_uses_uuid_when_provided(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
uuid = "GPU-12345678-1234-1234-1234-123456789abc"
mock_handle = mock.MagicMock()
mock_handle = mocker.MagicMock()
with (
mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": uuid}),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
mock.patch("vllm.third_party.pynvml.nvmlDeviceGetHandleByUUID", return_value=mock_handle) as mock_by_uuid,
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetComputeRunningProcesses", return_value=[]),
):
memory = get_process_gpu_memory(0)
assert memory == 0
mock_by_uuid.assert_called_once_with(uuid)
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": uuid})
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
mock_by_uuid = mocker.patch("vllm.third_party.pynvml.nvmlDeviceGetHandleByUUID", return_value=mock_handle)
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlDeviceGetComputeRunningProcesses", return_value=[])
def test_raises_on_invalid_uuid(self):
memory = get_process_gpu_memory(0)
assert memory == 0
mock_by_uuid.assert_called_once_with(uuid)
def test_raises_on_invalid_uuid(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
uuid = "GPU-invalid-uuid"
with (
mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": uuid}),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
mock.patch("vllm.third_party.pynvml.nvmlDeviceGetHandleByUUID", side_effect=Exception("Invalid UUID")),
):
with pytest.raises(RuntimeError, match="Failed to get NVML handle"):
get_process_gpu_memory(0)
mocker.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": uuid})
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
mocker.patch("vllm.third_party.pynvml.nvmlDeviceGetHandleByUUID", side_effect=Exception("Invalid UUID"))
def test_returns_none_on_nvml_init_failure(self):
with pytest.raises(RuntimeError, match="Failed to get NVML handle"):
get_process_gpu_memory(0)
def test_returns_none_on_nvml_init_failure(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import get_process_gpu_memory
with (
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit", side_effect=Exception("NVML unavailable")),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
):
result = get_process_gpu_memory(0)
assert result is None
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit", side_effect=Exception("NVML unavailable"))
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
result = get_process_gpu_memory(0)
assert result is None
class TestIsProcessScopedMemoryAvailable:
def test_returns_true_when_nvml_works(self):
def test_returns_true_when_nvml_works(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import is_process_scoped_memory_available
with (
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit"),
mock.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown"),
):
assert is_process_scoped_memory_available() is True
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlInit")
mocker.patch("vllm_omni.worker.gpu_memory_utils.nvmlShutdown")
assert is_process_scoped_memory_available() is True
def test_returns_false_when_nvml_fails(self):
def test_returns_false_when_nvml_fails(self, mocker: MockerFixture):
from vllm_omni.worker.gpu_memory_utils import is_process_scoped_memory_available
with mock.patch(
mocker.patch(
"vllm_omni.worker.gpu_memory_utils.nvmlInit",
side_effect=Exception("NVML unavailable"),
):
assert is_process_scoped_memory_available() is False
)
assert is_process_scoped_memory_available() is False