Rebase multi-API serving on latest main

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-16 12:13:04 +08:00
1 parent 78934753ff
commit dc3c7dbe16
25 files changed
+1832 -202

No files matched your search

@@ -8,12 +8,15 @@ import json
import os
import time
import types
from dataclasses import dataclass, field
import pytest
from vllm.v1.engine.utils import EngineZmqAddresses
from vllm_omni.diffusion.data import AttentionConfig
from vllm_omni.engine import async_omni_engine as async_omni_engine_module
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
from vllm_omni.engine.stage_init_utils import (
LogicalStageInitPlan,
ReplicaInitPlan,
@@ -26,6 +29,32 @@ from vllm_omni.engine.stage_runtime import StageRuntime
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@dataclass
class _FakeParallelConfig:
enable_fault_tolerance: bool = False
enable_elastic_ep: bool = False
data_parallel_size: int = 1
use_ray: bool = False
@dataclass
class _FakeVllmConfig:
parallel_config: _FakeParallelConfig = field(default_factory=_FakeParallelConfig)
@dataclass
class _FakeRuntimeConfig:
devices: str
@dataclass
class _FakeStageConfig:
stage_id: int
stage_type: str
engine_args: dict[str, object]
runtime: _FakeRuntimeConfig
def test_orchestrator_startup_timeout_warns_how_to_raise_limits(monkeypatch):
engine = object.__new__(AsyncOmniEngine)
engine.orchestrator_thread = types.SimpleNamespace(is_alive=lambda: True)
@@ -776,6 +805,246 @@ def test_stage_runtime_passes_log_stats_to_llm_replica_launch(monkeypatch):
assert captured["client_log_stats"] is True
def test_stage_runtime_attaches_external_llm_client_without_launching_engine(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
client_addresses = {
"input_address": "ipc://stage0-input-1",
"output_address": "ipc://stage0-output-1",
}
client_config = {
"client_count": 2,
"client_index": 1,
"stage_addresses": {0: {0: client_addresses}},
}
runtime = StageRuntime(
stage_configs=[],
model="dummy-model",
config_path="dummy-config",
stage_init_timeout=1,
async_chunk=False,
client_config=client_config,
)
parallel_config = _FakeParallelConfig()
vllm_config = _FakeVllmConfig(parallel_config=parallel_config)
plan = _make_llm_plan(0, stage_id=0, vllm_config=vllm_config).replicas[0]
captured: dict[str, object] = {}
stage_client = object()
def _capture_client(**kwargs):
captured.update(kwargs)
return stage_client
monkeypatch.setattr(runtime_mod.StageEngineCoreClientBase, "make_async_mp_client", _capture_client)
monkeypatch.setattr(
runtime_mod,
"launch_stage_replica",
lambda **_kwargs: pytest.fail("external client must not launch a stage engine"),
)
assert runtime._initialize_local_llm_replica(plan, stage_init_timeout=1) is stage_client
assert captured["client_addresses"] == client_addresses
assert captured["client_count"] == 2
assert captured["client_index"] == 1
assert not hasattr(parallel_config, "_api_process_count")
assert not hasattr(parallel_config, "_api_process_rank")
@pytest.mark.parametrize("stage_ids", [(0,), (0, 1)], ids=["single-stage", "multi-stage"])
@pytest.mark.parametrize("client_count", [1, 2])
def test_stage_runtime_launches_shared_engines_with_per_client_addresses(monkeypatch, stage_ids, client_count):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
stage_plans = []
for stage_id in stage_ids:
parallel_config = _FakeParallelConfig()
plan = _make_llm_plan(
stage_id,
stage_id=stage_id,
vllm_config=_FakeVllmConfig(parallel_config=parallel_config),
)
plan.replicas[0].engine_args_dict = {}
stage_plans.append(plan)
stage_plans[0].replicas[0].metadata.runtime_cfg = {"env": {"VLLM_OMNI_TEST_STAGE_RUNTIME_ENV": "enabled"}}
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: stage_plans)
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda stage_id, _cfg: str(stage_id))
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args: [])
events: list[tuple[str, int]] = []
captured_launch_kwargs: list[dict[str, object]] = []
captured_launch_env: list[str | None] = []
@contextlib.contextmanager
def _fake_launch_stage_replica(**kwargs):
stage_id = kwargs["stage_id"]
events.append(("enter", stage_id))
captured_launch_kwargs.append(kwargs)
captured_launch_env.append(os.environ.get("VLLM_OMNI_TEST_STAGE_RUNTIME_ENV"))
yield StageReplicaResources(
addresses=EngineZmqAddresses(
inputs=[f"ipc://stage{stage_id}-input-{idx}" for idx in range(client_count)],
outputs=[f"ipc://stage{stage_id}-output-{idx}" for idx in range(client_count)],
),
)
events.append(("exit", stage_id))
monkeypatch.setattr(runtime_mod, "launch_stage_replica", _fake_launch_stage_replica)
with runtime.launch_stage_engines(client_count) as launch:
expected = (
[("enter", stage_id) for stage_id in stage_ids]
if client_count > 1
else [(event, stage_id) for stage_id in stage_ids for event in ("enter", "exit")]
)
assert events == expected
for client_index, config in enumerate(launch.client_configs):
assert config["client_count"] == client_count
assert config["client_index"] == client_index
for stage_id in stage_ids:
assert config["stage_addresses"][stage_id][0] == {
"input_address": f"ipc://stage{stage_id}-input-{client_index}",
"output_address": f"ipc://stage{stage_id}-output-{client_index}",
}
assert events == (
[(event, stage_id) for event in ("enter", "exit") for stage_id in stage_ids] if client_count > 1 else expected
)
assert not hasattr(stage_plans[0].replicas[0].stage_vllm_config.parallel_config, "_api_process_count")
assert not hasattr(stage_plans[0].replicas[0].stage_vllm_config.parallel_config, "_api_process_rank")
assert all(
kwargs["watched_frontend_processes"] is (launch.watched_frontend_processes if client_count > 1 else None)
for kwargs in captured_launch_kwargs
)
assert captured_launch_env == ["enabled" if stage_id == 0 else None for stage_id in stage_ids]
assert os.environ.get("VLLM_OMNI_TEST_STAGE_RUNTIME_ENV") is None
@pytest.mark.parametrize("diffusion_stage_id", [0, 1], ids=["diffusion-only", "enginecore-to-diffusion"])
def test_stage_runtime_multi_api_rejects_diffusion_before_launch(monkeypatch, diffusion_stage_id):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
stage_plans = [
_make_llm_plan(stage_id, stage_id=stage_id, vllm_config=_FakeVllmConfig())
for stage_id in range(diffusion_stage_id)
]
stage_plans.append(_make_diffusion_plan(diffusion_stage_id, stage_id=diffusion_stage_id))
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: stage_plans)
monkeypatch.setattr(
runtime_mod,
"acquire_device_locks",
lambda *_args: pytest.fail("unsupported topology must fail before acquiring devices"),
)
monkeypatch.setattr(
runtime_mod,
"launch_stage_replica",
lambda **_kwargs: pytest.fail("unsupported topology must not partially launch EngineCore stages"),
)
with pytest.raises(ValueError, match="diffusion stage\\(s\\) are not supported"):
with runtime.launch_stage_engines(2):
pytest.fail("unsupported topology must not yield a launch handle")
def test_stage_runtime_multi_api_maps_stage_devices_from_launcher_visibility(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
stage_plan = _make_llm_plan(
0,
stage_id=0,
vllm_config=_FakeVllmConfig(),
)
replica = stage_plan.replicas[0]
replica.engine_args_dict = {}
replica.metadata.runtime_cfg = {"devices": "0"}
device_env = runtime_mod.current_omni_platform.device_control_env_var
monkeypatch.setenv(device_env, "5")
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: [stage_plan])
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args: [])
captured_devices: list[str | None] = []
@contextlib.contextmanager
def _fake_launch_stage_replica(**kwargs):
captured_devices.append(kwargs["stage_visible_devices"])
yield StageReplicaResources(
addresses=EngineZmqAddresses(
inputs=["ipc://input-0", "ipc://input-1"],
outputs=["ipc://output-0", "ipc://output-1"],
),
)
monkeypatch.setattr(runtime_mod, "launch_stage_replica", _fake_launch_stage_replica)
with runtime.launch_stage_engines(2):
pass
assert captured_devices == ["5"]
assert os.environ[device_env] == "5"
def test_stage_runtime_multi_api_rejects_elastic_ep(monkeypatch):
runtime = _make_stage_runtime()
stage_plan = _make_llm_plan(
0,
stage_id=0,
vllm_config=_FakeVllmConfig(parallel_config=_FakeParallelConfig(enable_elastic_ep=True)),
)
stage_plan.replicas[0].engine_args_dict = {}
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: [stage_plan])
with pytest.raises(ValueError, match="enable-elastic-ep"):
with runtime.launch_stage_engines(2):
pass
def test_stage_runtime_multi_api_failure_shuts_down_before_exceptional_context_exit(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
stage_plan = _make_llm_plan(0, stage_id=0, vllm_config=_FakeVllmConfig())
stage_plan.replicas[0].engine_args_dict = {}
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: [stage_plan])
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda *_args: None)
monkeypatch.setattr(runtime_mod, "acquire_device_locks", lambda *_args: [])
events: list[str] = []
class _Manager:
def shutdown(self) -> None:
events.append("shutdown")
@contextlib.contextmanager
def _fake_launch_stage_replica(**_kwargs):
try:
yield StageReplicaResources(
manager=_Manager(),
addresses=EngineZmqAddresses(
inputs=["tcp://127.0.0.1:0", "tcp://127.0.0.1:1"],
outputs=["tcp://127.0.0.1:2", "tcp://127.0.0.1:3"],
),
)
except RuntimeError as exc:
events.append(f"exceptional-exit:{exc}")
raise
else:
events.append("normal-exit")
monkeypatch.setattr(runtime_mod, "launch_stage_replica", _fake_launch_stage_replica)
with pytest.raises(RuntimeError, match="deferred TCP addresses"):
with runtime.launch_stage_engines(2):
pass
assert events == [
"shutdown",
"exceptional-exit:Stage 0 returned deferred TCP addresses; multi-API launch requires fixed ports or IPC addresses",
]
def test_stage_runtime_passes_log_stats_to_output_processor(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
@@ -1022,7 +1291,7 @@ def test_initialize_local_llm_replica_passes_stage_init_timeout_to_complete_stag
num_replicas=1,
launch_mode="local",
stage_cfg=types.SimpleNamespace(engine_args={}, runtime=types.SimpleNamespace(devices="0")),
metadata=types.SimpleNamespace(stage_id=0, runtime_cfg={"devices": "0"}),
metadata=types.SimpleNamespace(stage_id=0, stage_type="llm", runtime_cfg={"devices": "0"}),
stage_connector_spec={},
omni_kv_connector=(None, None, None),
stage_vllm_config=fake_vllm_config,
@@ -1829,3 +2098,99 @@ def test_port_from_zmq_address_parsing():
assert _port_from_zmq_address(None) is None
assert _port_from_zmq_address("ipc:///tmp/sock") is None
assert _port_from_zmq_address("tcp://host:not-a-port") is None
@pytest.mark.parametrize("parallel", [False, True])
@pytest.mark.parametrize("failure", ["ready", "attach", None])
def test_single_api_common_launch_ownership_and_rollback(monkeypatch, parallel, failure):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
runtime._parallel_stage_init = parallel
plan = _make_llm_plan(0, stage_id=0, vllm_config=_FakeVllmConfig()).replicas[0]
plan.engine_args_dict = {}
events = []
class Manager:
def shutdown(self):
events.append("shutdown")
manager = Manager()
addresses = EngineZmqAddresses(inputs=["ipc://input"], outputs=["ipc://output"])
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda *_: "0")
def acquire(*_):
events.append("lock")
return [42]
monkeypatch.setattr(runtime_mod, "acquire_device_locks", acquire)
monkeypatch.setattr(runtime_mod, "release_device_locks", lambda _: events.append("unlock"))
@contextlib.contextmanager
def launch(**kwargs):
assert kwargs["num_api_servers"] == 1
assert kwargs["omni_parallel_stage_init"] is parallel
events.append("spawn")
yield StageReplicaResources(manager=manager, addresses=addresses)
assert runtime._replica_launch_lock.locked() is False
events.append("ready")
if failure == "ready":
raise RuntimeError("ready")
def attach(**kwargs):
assert "ready" in events
assert kwargs["engine_manager"] is manager
events.append("attach")
if failure == "attach":
raise RuntimeError("attach")
return manager
monkeypatch.setattr(runtime_mod, "launch_stage_replica", launch)
monkeypatch.setattr(runtime_mod.StageEngineCoreClientBase, "make_async_mp_client", attach)
if failure:
with pytest.raises(RuntimeError, match=failure):
runtime._initialize_local_llm_replica(plan, 1)
assert events.count("shutdown") == 1
else:
assert runtime._initialize_local_llm_replica(plan, 1) is manager
assert "shutdown" not in events
assert events.count("lock") == events.count("unlock") == (0 if parallel else 1)
@pytest.mark.parametrize("client_count", [1, 2])
def test_common_launch_parallel_admission_before_spawn(monkeypatch, client_count):
import vllm_omni.engine.stage_runtime as runtime_mod
runtime = _make_stage_runtime()
runtime._parallel_stage_init = True
plan = _make_llm_plan(0, stage_id=0, vllm_config=_FakeVllmConfig())
plan.replicas[0].engine_args_dict = {}
events = []
monkeypatch.setattr(runtime, "_prepare_stage_plans", lambda: [plan])
monkeypatch.setattr(runtime, "_reject_unguardable_executors", lambda _: events.append("guard"))
monkeypatch.setattr(runtime, "_run_stage_admission", lambda _: events.append("admit"))
monkeypatch.setattr(runtime, "_resolve_replica_physical_devices", lambda *_: "0")
def forbidden_lock(*_):
pytest.fail("parallel initialization must use child phase locks")
monkeypatch.setattr(runtime_mod, "acquire_device_locks", forbidden_lock)
@contextlib.contextmanager
def launch(**kwargs):
assert events == ["guard", "admit"]
assert kwargs["omni_parallel_stage_init"] is True
events.append("spawn")
yield StageReplicaResources(
addresses=EngineZmqAddresses(
inputs=[f"ipc://in{i}" for i in range(client_count)],
outputs=[f"ipc://out{i}" for i in range(client_count)],
)
)
assert not runtime._replica_launch_lock.locked()
events.append("ready")
monkeypatch.setattr(runtime_mod, "launch_stage_replica", launch)
with runtime.launch_stage_engines(client_count):
pass
assert events == ["guard", "admit", "spawn", "ready"]
+26 -13
View File
@@ -18,6 +18,7 @@ import types
from unittest import mock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.engine import stage_init_utils
from vllm_omni.engine.stage_init_utils import (
@@ -203,7 +204,7 @@ def test_build_vllm_config_fails_before_engine_config_on_mismatch():
get_class.assert_not_called()
def test_build_vllm_config_proceeds_on_consistent_layout():
def test_build_vllm_config_proceeds_on_consistent_layout(mocker: MockerFixture) -> None:
"""The guard must not false-positive: a consistent single-GPU stage flows past
it and reaches ``create_engine_config`` / ``Executor.get_class`` as usual."""
stage = _stage(1, devices="1")
@@ -212,18 +213,30 @@ def test_build_vllm_config_proceeds_on_consistent_layout():
model_config=types.SimpleNamespace(hf_config=types.SimpleNamespace()),
)
sentinel_executor = object()
with (
mock.patch.object(
stage_init_utils.OmniEngineArgs, "create_engine_config", return_value=fake_config
) as create_engine_config,
mock.patch.object(stage_init_utils.Executor, "get_class", return_value=sentinel_executor),
mock.patch.object(stage_init_utils.OmniINCConfig, "maybe_upgrade", side_effect=lambda quant: quant),
):
vllm_config, executor_class = build_vllm_config(
stage,
model="dummy-model",
engine_args_dict={"tensor_parallel_size": 1},
)
captured_api_process_config: dict[str, int] = {}
def capture_create_engine_config(engine_args, *args, **kwargs):
captured_api_process_config["count"] = engine_args._api_process_count
captured_api_process_config["rank"] = engine_args._api_process_rank
return fake_config
create_engine_config = mocker.patch.object(
stage_init_utils.OmniEngineArgs,
"create_engine_config",
autospec=True,
side_effect=capture_create_engine_config,
)
mocker.patch.object(stage_init_utils.Executor, "get_class", return_value=sentinel_executor)
mocker.patch.object(stage_init_utils.OmniINCConfig, "maybe_upgrade", side_effect=lambda quant: quant)
vllm_config, executor_class = build_vllm_config(
stage,
model="dummy-model",
engine_args_dict={"tensor_parallel_size": 1},
api_process_count=3,
api_process_rank=2,
)
create_engine_config.assert_called_once()
assert vllm_config is fake_config
assert executor_class is sentinel_executor
assert captured_api_process_config == {"count": 3, "rank": 2}
@@ -42,7 +42,10 @@ from __future__ import annotations
import asyncio
import json
import signal
from argparse import Namespace
from contextlib import asynccontextmanager
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
@@ -59,6 +62,66 @@ from vllm_omni.entrypoints.serve.utils import errors as serve_errors
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@dataclass
class _FakeParallelConfig:
_api_process_rank: int
@dataclass
class _FakeVllmConfig:
parallel_config: _FakeParallelConfig
def test_omni_api_worker_sets_parent_death_signal_before_validation(mocker):
args = _minimal_args(_omni_stage_client_configs=[{"stage_addresses": {}}])
death_signal = mocker.patch.object(api_server, "set_death_signal")
with pytest.raises(RuntimeError, match="client configuration"):
api_server.run_omni_api_server_worker_proc(
"127.0.0.1:8000",
mocker.Mock(),
args,
{"client_count": 2, "client_index": -1},
)
death_signal.assert_called_once_with(signal.SIGTERM)
@pytest.mark.asyncio
async def test_omni_run_server_worker_forwards_client_config(monkeypatch):
captured: dict[str, object] = {}
@asynccontextmanager
async def fake_build_async_omni(*_args, **kwargs):
captured.update(kwargs)
yield _FakeEngineClient(vllm_config=_FakeVllmConfig(_FakeParallelConfig(_api_process_rank=1)))
monkeypatch.setattr(api_server, "build_async_omni", fake_build_async_omni)
monkeypatch.setattr(api_server, "build_openai_app", lambda *_args: FastAPI())
monkeypatch.setattr(api_server, "omni_init_app_state", lambda *_args: asyncio.sleep(0))
monkeypatch.setattr(api_server, "shutdown_unsupported_routes", lambda *_args: None)
async def fake_storage_start() -> None:
return None
monkeypatch.setattr(api_server.STORAGE_MANAGER, "start", fake_storage_start)
async def fake_serve_http(app, **_kwargs):
assert app.state.api_server_count == 2
task = asyncio.create_task(asyncio.sleep(0))
await asyncio.sleep(0)
return task
monkeypatch.setattr(api_server, "serve_http", fake_serve_http)
args = _minimal_args(model="dummy")
config = {"client_count": 2, "client_index": 1, "stage_addresses": {}}
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), args, config)
assert captured["client_config"] is config
# FastAPI/Starlette auto-adds HEAD on every GET route. Including HEAD here
# would invent ("HEAD", path) entries for every GET and fail the current census.
# TODO: extend this set and the expected census if @router.head / @router.options appear.
@@ -708,6 +771,38 @@ def test_speech_without_handler_preserves_not_found_http_error() -> None:
assert exc_info.value.detail == "The model does not support Speech API"
@pytest.mark.asyncio
async def test_multi_api_rejects_runtime_voice_upload() -> None:
app = FastAPI()
app.state.api_server_count = 2
raw_request = _request_for(app, method="POST", path="/v1/audio/voices")
with pytest.raises(HTTPException) as exc_info:
await api_server.upload_voice(
raw_request,
audio_sample=None,
speaker_embedding=None,
consent="consent-id",
name="probe",
)
assert exc_info.value.status_code == 409
assert "process-local frontend state" in exc_info.value.detail
@pytest.mark.asyncio
async def test_multi_api_rejects_sleep_route() -> None:
app = FastAPI()
app.state.api_server_count = 2
raw_request = _request_for(app, method="POST", path="/v1/omni/sleep")
with pytest.raises(HTTPException) as exc_info:
await api_server.omni_sleep(api_server.OmniSleepRequest(stage_ids=[0]), raw_request)
assert exc_info.value.status_code == 409
assert "process-local frontend state" in exc_info.value.detail
def test_engine_error_json_response_includes_request_and_stage_fields(monkeypatch) -> None:
"""Lock shared ``EngineGenerateError`` JSON fields used by Omni handlers.
@@ -931,3 +1026,57 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
must_be_wired=_MULTISTAGE_MUST_BE_WIRED,
must_be_none=_MULTISTAGE_MUST_BE_NONE,
)
@pytest.mark.parametrize("count", [None, 0, -1, "2", True])
def test_process_local_guard_rejects_unknown_topology(count):
app = FastAPI()
if count is not None:
app.state.api_server_count = count
# CLI args cannot substitute for missing/invalid actual runtime state.
app.state.args = Namespace(api_server_count=1)
with pytest.raises(HTTPException) as exc:
api_server._reject_process_local_state_with_multiple_api_workers(_request_for(app), "Sleep")
assert exc.value.status_code == 503
@pytest.mark.parametrize(
"requested,config,expected",
[(None, None, 1), (1, None, 1), (2, {"client_count": 2}, 2), (None, {"client_count": 2}, 2)],
)
def test_api_topology_resolved_from_launch(requested, config, expected):
assert api_server._resolve_api_server_count(Namespace(api_server_count=requested), config) == expected
@pytest.mark.parametrize("requested,config", [(2, None), (2, {}), (2, {"client_count": 1}), (2, {"client_count": "2"})])
def test_api_topology_rejects_inconsistent_launch(requested, config):
with pytest.raises(ValueError):
api_server._resolve_api_server_count(Namespace(api_server_count=requested), config)
@pytest.mark.parametrize("count,status", [(2, 409), (None, 503)])
@pytest.mark.parametrize(
"method,path",
[
("POST", "/v1/videos"),
("GET", "/v1/videos"),
("GET", "/v1/videos/job"),
("GET", "/v1/videos/job/content"),
("DELETE", "/v1/videos/job"),
],
)
def test_video_lifecycle_guard_precedes_processing(monkeypatch, count, status, method, path):
from fastapi.testclient import TestClient
app = FastAPI()
if count is not None:
app.state.api_server_count = count
app.include_router(api_server.router)
async def forbidden_form():
pytest.fail("video form processing must not run before topology rejection")
app.dependency_overrides[api_server._parse_video_form] = forbidden_form
with TestClient(app) as client:
response = client.request(method, path)
assert response.status_code == status
@@ -20,6 +20,7 @@ def _make_app(engine_client):
from vllm_omni.entrypoints.openai.api_server import router
app = FastAPI()
app.state.api_server_count = 1
app.include_router(router)
app.state.engine_client = engine_client
app.state.sleeping_stages = set()
@@ -253,6 +254,7 @@ def pure_diffusion_app(pure_diffusion_engine, mocker):
)
app = FastAPI()
app.state.api_server_count = 1
app.include_router(router)
mocker.patch(
@@ -3793,6 +3793,7 @@ def test_api_server_create_speech_wraps_error_response_status(mocker: MockerFixt
def _make_api_server_request(handler, *, method: str = "POST", path: str = "/v1/audio/voices") -> Request:
app = FastAPI()
app.state.openai_serving_speech = handler
app.state.api_server_count = 1
scope = {
"type": "http",
"app": app,
@@ -345,6 +345,7 @@ async def test_server_worker_keeps_engine_alive_until_http_shutdown(monkeypatch)
@pytest.fixture
def test_client():
app = FastAPI()
app.state.api_server_count = 1
app.include_router(router)
app.state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion(
diffusion_engine=FakeAsyncOmni(),
@@ -1629,6 +1630,7 @@ def test_action_extraction_accepts_multimodal_actions_payload():
def test_missing_handler_returns_503():
app = FastAPI()
app.state.api_server_count = 1
app.include_router(router)
app.state.openai_serving_video = None
client = TestClient(app)
@@ -2541,6 +2543,7 @@ def test_cosmos3_control_upload_rejects_invalid_size(control_bytes, message, tes
def test_sync_missing_handler_returns_503():
app = FastAPI()
app.state.api_server_count = 1
app.include_router(router)
app.state.openai_serving_video = None
client = TestClient(app)
+259
View File
@@ -6,12 +6,17 @@
from __future__ import annotations
import argparse
from dataclasses import dataclass
from multiprocessing.reduction import ForkingPickler
from types import SimpleNamespace
import pytest
from pytest_mock import MockerFixture
from vllm.v1.engine.utils import EngineZmqAddresses
from vllm_omni.config.resolver import OmniConfigResolution
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
from vllm_omni.engine.stage_runtime import StageEngineLaunch
from vllm_omni.entrypoints.cli.serve import (
OmniServeCommand,
_parse_stage_overrides,
@@ -23,6 +28,31 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingName
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@dataclass(frozen=True)
class _FakeProcess:
sentinel: str
exitcode: int | None
name: str
pid: int
@dataclass
class _FakeProcessOwner:
processes: list[_FakeProcess]
@dataclass
class _FakeStageEngineArgs:
async_chunk: bool
enable_sleep_mode: bool
@dataclass
class _FakeStageConfig:
stage_id: int
engine_args: _FakeStageEngineArgs
def _resolved(*stages: SimpleNamespace) -> OmniConfigResolution:
return OmniConfigResolution(config_path="/fake/stages.yaml", stage_configs=tuple(stages))
@@ -42,6 +72,8 @@ def test_serve_parser_accepts_no_async_chunk_and_marks_it_explicit() -> None:
explicit = args.get_explicit_kwargs_dict()
assert args.get_explicit_kwargs_dict()
assert not explicit["async_chunk"]
assert (args.api_server_count or 1) == 1
assert "api_server_count" not in explicit
def _parse_serve_args(argv: list[str]) -> TrackingNamespace:
@@ -167,6 +199,214 @@ def test_serve_parser_rejects_invalid_video_output_transport(value: str) -> None
parser.parse_args(["serve", "fake-model", "--omni", "--video-output-transport", value])
def test_tracking_namespace_is_picklable_for_spawned_api_workers() -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
OmniServeCommand().subparser_init(subparsers)
args = parser.parse_args(["serve", "fake-model", "--omni", "--api-server-count", "2"])
args._omni_stage_client_configs = [{"stage_addresses": {0: {0: {"input_address": "ipc://input"}}}}]
restored = ForkingPickler.loads(ForkingPickler.dumps(args))
assert restored.api_server_count == 2
assert restored.get_explicit_kwargs_dict()["api_server_count"] == 2
assert restored._omni_stage_client_configs == args._omni_stage_client_configs
def test_serve_validate_rejects_multiple_api_servers_for_diffusion(mocker: MockerFixture) -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
cmd = OmniServeCommand()
cmd.subparser_init(subparsers)
args = parser.parse_args(["serve", "fake-diffusion-model", "--omni", "--api-server-count", "2"])
mocker.patch("vllm_omni.diffusion.utils.hf_utils.is_diffusion_model", return_value=True)
validate = mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
with pytest.raises(ValueError, match="not supported for diffusion"):
cmd.validate(args)
validate.assert_not_called()
def test_serve_validate_rejects_sleep_mode_with_multiple_api_servers(mocker: MockerFixture) -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
cmd = OmniServeCommand()
cmd.subparser_init(subparsers)
args = parser.parse_args(["serve", "fake-model", "--omni", "--api-server-count", "2", "--enable-sleep-mode"])
mocker.patch("vllm_omni.diffusion.utils.hf_utils.is_diffusion_model", return_value=False)
mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
with pytest.raises(ValueError, match="enable-sleep-mode"):
cmd.validate(args)
def test_build_multi_api_stage_runtime_matches_current_constructor(mocker: MockerFixture) -> None:
from vllm_omni.entrypoints.cli import serve as serve_module
args = TrackingNamespace(
argparse.Namespace(
model="dummy-model",
model_tag=None,
stage_init_timeout=1,
tokenizer=None,
disable_log_stats=False,
),
frozenset({"model"}),
)
stage_config = _FakeStageConfig(
stage_id=0,
engine_args=_FakeStageEngineArgs(async_chunk=False, enable_sleep_mode=False),
)
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", return_value="dummy-model")
mocker.patch(
"vllm_omni.config.config_factory.with_trust_remote_code_override",
side_effect=lambda kwargs, _trust: kwargs,
)
mocker.patch("vllm_omni.entrypoints.utils.parse_stage_overrides", return_value={})
mocker.patch(
"vllm_omni.config.resolver.resolve_omni_config",
return_value=OmniConfigResolution(config_path="dummy-config", stage_configs=(stage_config,)),
)
runtime = serve_module._build_multi_api_stage_runtime(args, 2)
assert runtime._client_count == 1
assert runtime._client_index == 0
def test_build_multi_api_stage_runtime_rejects_sleep_enabled_in_stage_config(mocker: MockerFixture) -> None:
from vllm_omni.entrypoints.cli import serve as serve_module
args = TrackingNamespace(
argparse.Namespace(
model="dummy-model",
model_tag=None,
stage_init_timeout=1,
tokenizer=None,
disable_log_stats=False,
),
frozenset({"model"}),
)
stage_config = _FakeStageConfig(
stage_id=3,
engine_args=_FakeStageEngineArgs(async_chunk=False, enable_sleep_mode=True),
)
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", return_value="dummy-model")
mocker.patch(
"vllm_omni.config.config_factory.with_trust_remote_code_override",
side_effect=lambda kwargs, _trust: kwargs,
)
mocker.patch("vllm_omni.entrypoints.utils.parse_stage_overrides", return_value={})
mocker.patch(
"vllm_omni.config.resolver.resolve_omni_config",
return_value=OmniConfigResolution(config_path="dummy-config", stage_configs=(stage_config,)),
)
with pytest.raises(ValueError, match=r"sleep mode.*stage\(s\) \[3\]"):
serve_module._build_multi_api_stage_runtime(args, 2)
def test_run_multi_api_server_omni_starts_workers_after_shared_engine_launch(mocker: MockerFixture) -> None:
from vllm_omni.entrypoints.cli import serve as serve_module
args = TrackingNamespace(
argparse.Namespace(api_server_count=2, shutdown_timeout=1),
frozenset({"api_server_count", "shutdown_timeout"}),
)
socket = mocker.Mock()
primary_addresses = EngineZmqAddresses(
inputs=["ipc://input-0", "ipc://input-1"],
outputs=["ipc://output-0", "ipc://output-1"],
)
engine_launch = StageEngineLaunch(
client_configs=[{"client_count": 2, "client_index": index, "stage_addresses": {}} for index in range(2)],
resources=[StageReplicaResources(addresses=primary_addresses)],
)
runtime = mocker.MagicMock()
runtime.launch_stage_engines.return_value.__enter__.return_value = engine_launch
runtime.launch_stage_engines.return_value.__exit__.return_value = False
manager = mocker.Mock()
manager.processes = [mocker.Mock()]
manager.gather_actual_addresses.return_value = (primary_addresses.inputs, primary_addresses.outputs)
mocker.patch.object(serve_module, "_build_multi_api_stage_runtime", return_value=runtime)
mocker.patch.object(serve_module, "_wait_for_multi_api_server_completion")
mocker.patch("signal.signal")
mocker.patch("vllm.entrypoints.openai.api_server.setup_server", return_value=("127.0.0.1:8000", socket))
mocker.patch("vllm.v1.metrics.prometheus.setup_multiprocess_prometheus")
start_manager = mocker.patch.object(serve_module, "_start_api_server_process_manager", return_value=manager)
serve_module.run_multi_api_server_omni(args)
assert args._omni_stage_client_configs is engine_launch.client_configs
start_manager.assert_called_once()
manager_kwargs = start_manager.call_args.kwargs
assert manager_kwargs["cleanup_timeout"] == 1
assert manager_kwargs["num_servers"] == 2
assert manager_kwargs["input_addresses"] == ["ipc://input-0", "ipc://input-1"]
assert manager_kwargs["target_server_fn"] is serve_module.run_omni_api_server_worker_proc
assert engine_launch.watched_frontend_processes == manager.processes
runtime.shutdown.assert_called_once_with()
socket.close.assert_called_once_with()
def test_start_api_server_process_manager_cleans_up_partial_start(mocker: MockerFixture) -> None:
from vllm_omni.entrypoints.cli import serve as serve_module
first_process = mocker.Mock(name="first_process")
second_process = mocker.Mock(name="second_process")
first_process.pid = 1234
second_process.pid = None
second_process.start.side_effect = RuntimeError("spawn failed")
parent_pipes = [mocker.Mock(name="parent_pipe_0"), mocker.Mock(name="parent_pipe_1")]
child_pipes = [mocker.Mock(name="child_pipe_0"), mocker.Mock(name="child_pipe_1")]
spawn_context = mocker.Mock()
spawn_context.Process.side_effect = [first_process, second_process]
spawn_context.Pipe.side_effect = list(zip(parent_pipes, child_pipes))
mocker.patch("vllm.v1.utils.multiprocessing.get_context", return_value=spawn_context)
shutdown = mocker.patch("vllm.v1.utils.shutdown")
with pytest.raises(RuntimeError, match="spawn failed"):
serve_module._start_api_server_process_manager(
cleanup_timeout=1.5,
listen_address="127.0.0.1:8000",
sock=mocker.Mock(),
args=argparse.Namespace(),
num_servers=2,
input_addresses=["ipc://input-0", "ipc://input-1"],
output_addresses=["ipc://output-0", "ipc://output-1"],
target_server_fn=mocker.Mock(),
)
first_process.start.assert_called_once_with()
second_process.start.assert_called_once_with()
for pipe in parent_pipes:
pipe.close.assert_called_once_with()
shutdown.assert_called_once_with([first_process], timeout=1.5)
def test_wait_for_multi_api_server_completion_rejects_engine_failure(
mocker: MockerFixture,
) -> None:
from vllm_omni.entrypoints.cli import serve as serve_module
api_process = _FakeProcess(sentinel="api-0", exitcode=None, name="api-0", pid=10)
engine_process = _FakeProcess(sentinel="engine-0", exitcode=4, name="engine-0", pid=20)
manager = _FakeProcessOwner(processes=[api_process])
engine_launch = StageEngineLaunch(
client_configs=[],
resources=[StageReplicaResources(manager=_FakeProcessOwner(processes=[engine_process]))],
)
mocker.patch("multiprocessing.connection.wait", return_value=["engine-0"])
with pytest.raises(RuntimeError, match=r"Shared stage engine process engine-0 .* exited with code 4"):
serve_module._wait_for_multi_api_server_completion(manager, engine_launch)
def test_serve_parser_rejects_stage_configs_path() -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
@@ -754,3 +994,22 @@ def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixtur
with pytest.raises(RuntimeError, match=r"exited with code 137"):
run_headless(_make_headless_args(stage_id=1))
@pytest.mark.parametrize("name", ["unfiltered_ns", "explicit_keys", "missing", "__dict__"])
def test_tracking_namespace_uninitialized_access_raises_attribute_error(name):
namespace = object.__new__(TrackingNamespace)
with pytest.raises(AttributeError):
getattr(namespace, name)
@pytest.mark.parametrize("deep", [False, True])
def test_tracking_namespace_copy_preserves_tracking(deep):
import copy
namespace = TrackingNamespace(
argparse.Namespace(model="example", api_server_count=2), frozenset({"api_server_count"})
)
restored = copy.deepcopy(namespace) if deep else copy.copy(namespace)
assert restored.get_explicit_kwargs_dict() == {"api_server_count": 2}
assert restored.model == "example"
+18
View File
@@ -529,3 +529,21 @@ class TestCumulativeStreamingCoercion:
assert all([isinstance(r, SamplingParams) for r in result])
assert result[0].output_kind == RequestOutputKind.DELTA
assert result[1].output_kind == RequestOutputKind.DELTA
@pytest.mark.parametrize("api_server_count", [1, 2])
def test_prepare_stage_config_inputs_filters_frontend_options(api_server_count):
from vllm_omni.entrypoints.utils import prepare_stage_config_inputs
kwargs = {
"api_server_count": api_server_count,
"disable_log_stats": True,
"model_tag": "example",
"max_num_seqs": 8,
"stage_overrides": '{"0": {"max_num_seqs": 4}}',
}
inputs = prepare_stage_config_inputs("example", kwargs, trust_remote_code=None)
assert inputs.kwargs == {"max_num_seqs": 8}
assert inputs.stage_overrides == {"0": {"max_num_seqs": 4}}
assert kwargs["api_server_count"] == api_server_count