[FEAT] support multi-stage deployment (#2396)
Signed-off-by: ZhengWG <zwg0606@gmail.com> Signed-off-by: Zheng Wengang <zwg0606@gmail.com> Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Signed-off-by: yinpe <11810305@mail.sustech.edu.cn> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: yinpe <11810305@mail.sustech.edu.cn> Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
55 files changed
+4684
-3407
No files matched your search
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -94,3 +95,24 @@ def test_inline_shutdown(client, mock_engine):
|
||||
|
||||
assert client._shutting_down
|
||||
mock_engine.close.assert_called_once()
|
||||
|
||||
|
||||
def test_inline_client_requires_replica_id(mock_engine):
|
||||
metadata = SimpleNamespace(
|
||||
stage_id=0,
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
default_sampling_params={},
|
||||
requires_multimodal_data=False,
|
||||
custom_process_input_func=None,
|
||||
engine_input_source=[],
|
||||
)
|
||||
with patch.object(InlineStageDiffusionClient, "_enrich_config"):
|
||||
od_config = MagicMock(spec=OmniDiffusionConfig)
|
||||
with pytest.raises(AttributeError, match="replica_id"):
|
||||
InlineStageDiffusionClient(
|
||||
model="test_model",
|
||||
od_config=od_config,
|
||||
metadata=metadata,
|
||||
batch_size=1,
|
||||
)
|
||||
@@ -621,6 +621,29 @@ class TestStageDiffusionClientErrorPropagation:
|
||||
assert client._shutting_down is True
|
||||
assert client._engine_dead is True
|
||||
|
||||
def test_initialize_client_requires_replica_id(self):
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
|
||||
client = object.__new__(StageDiffusionClient)
|
||||
metadata = SimpleNamespace(
|
||||
stage_id=0,
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
default_sampling_params=None,
|
||||
requires_multimodal_data=False,
|
||||
custom_process_input_func=None,
|
||||
engine_input_source=[],
|
||||
)
|
||||
|
||||
with pytest.raises(AttributeError, match="replica_id"):
|
||||
client._initialize_client(
|
||||
metadata,
|
||||
"tcp://req",
|
||||
"tcp://resp",
|
||||
proc=None,
|
||||
batch_size=1,
|
||||
)
|
||||
|
||||
|
||||
# ───────── monitor thread & death sentinel integration tests ─────────
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving.
|
||||
|
||||
This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
|
||||
ROUTE_STRESS_REQUESTS = 6
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
|
||||
|
||||
test_params = [
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=MULTI_REPLICA_DEPLOY,
|
||||
server_args=["--disable-log-stats"],
|
||||
use_stage_cli=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_multi_stage_diffusion_uses_multi_replica_dit_stage(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": dummy_messages_from_mix_data(
|
||||
content_text="<|im_start|>A small red cube on a white table.<|im_end|>"
|
||||
),
|
||||
"modalities": ["image"],
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
responses = openai_client.send_diffusion_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
|
||||
|
||||
assert len(responses) == ROUTE_STRESS_REQUESTS
|
||||
assert all(response.success for response in responses)
|
||||
assert all(response.images is not None and response.images[0].size == (512, 512) for response in responses)
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
Core-model CI guard for Qwen3-Omni multi-replica stage-pool routing on 4 GPUs.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||
from tests.helpers.runtime import OmniResponse, OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe_multi_replicas_4gpu.yaml")
|
||||
ROUTE_STRESS_REQUESTS = 6
|
||||
MIXED_MODAL_REQUESTS = 4
|
||||
|
||||
test_params = [
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=MULTI_REPLICA_DEPLOY,
|
||||
server_args=["--disable-log-stats"],
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _system_prompt() -> dict[str, object]:
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _text_messages() -> list[dict[str, object]]:
|
||||
return dummy_messages_from_mix_data(
|
||||
system_prompt=_system_prompt(),
|
||||
content_text="What is the capital of China? Answer in one short sentence.",
|
||||
)
|
||||
|
||||
|
||||
def _mixed_messages() -> list[dict[str, object]]:
|
||||
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
|
||||
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
|
||||
return dummy_messages_from_mix_data(
|
||||
system_prompt=_system_prompt(),
|
||||
video_data_url=video_data_url,
|
||||
image_data_url=image_data_url,
|
||||
audio_data_url=audio_data_url,
|
||||
content_text="What is recited in the audio? What is in this image? Describe the video briefly.",
|
||||
)
|
||||
|
||||
|
||||
def _assert_batch_size(responses: list[OmniResponse], expected: int) -> None:
|
||||
assert len(responses) == expected, f"Expected {expected} responses, got {len(responses)}"
|
||||
assert all(resp.success for resp in responses), "At least one request failed"
|
||||
assert all(resp.e2e_latency is not None and resp.e2e_latency > 0 for resp in responses), "Missing request latency"
|
||||
|
||||
|
||||
def _assert_text_outputs(responses: list[OmniResponse]) -> None:
|
||||
assert all(resp.text_content is not None for resp in responses), "Missing text output"
|
||||
assert all(resp.audio_bytes is None for resp in responses), "Text-only request unexpectedly produced audio"
|
||||
|
||||
|
||||
def _assert_audio_outputs(responses: list[OmniResponse], *, expect_text: bool) -> None:
|
||||
assert all(resp.audio_bytes is not None and len(resp.audio_bytes) > 128 for resp in responses), (
|
||||
"Missing audio output"
|
||||
)
|
||||
if expect_text:
|
||||
assert all(resp.text_content is not None for resp in responses), "Missing text output"
|
||||
else:
|
||||
assert all(not (resp.text_content or "").strip() for resp in responses), (
|
||||
"Audio-only request unexpectedly produced text"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_only_batch_uses_multi_replica_talker(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": _text_messages(),
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
|
||||
responses = openai_client.send_omni_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
|
||||
_assert_batch_size(responses, ROUTE_STRESS_REQUESTS)
|
||||
_assert_text_outputs(responses)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": _text_messages(),
|
||||
"stream": True,
|
||||
"modalities": ["audio"],
|
||||
}
|
||||
|
||||
responses = openai_client.send_omni_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
|
||||
_assert_batch_size(responses, ROUTE_STRESS_REQUESTS)
|
||||
_assert_audio_outputs(responses, expect_text=False)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_mixed_modal_stream_batch_generates_text_and_audio(omni_server, openai_client) -> None:
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": _mixed_messages(),
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
responses = openai_client.send_omni_request(request_config, request_num=MIXED_MODAL_REQUESTS)
|
||||
_assert_batch_size(responses, MIXED_MODAL_REQUESTS)
|
||||
_assert_audio_outputs(responses, expect_text=True)
|
||||
@@ -59,7 +59,7 @@ def test_build_add_request_message_preserves_additional_information(mocker: Mock
|
||||
assert request.additional_information is not None
|
||||
assert request.additional_information.entries["text"].list_data == ["hello world"]
|
||||
assert request.additional_information.entries["speaker"].list_data == ["vivian"]
|
||||
output_processor.add_request.assert_called_once()
|
||||
output_processor.add_request.assert_not_called()
|
||||
|
||||
|
||||
def test_build_add_request_message_with_resumable_streaming(mocker: MockerFixture):
|
||||
|
||||
@@ -1,15 +1,138 @@
|
||||
import importlib
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
LogicalStageInitPlan,
|
||||
ReplicaInitPlan,
|
||||
build_stage0_input_processor,
|
||||
compute_replica_layout,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_llm_metadata(
|
||||
stage_id: int,
|
||||
*,
|
||||
replica_id: int = 0,
|
||||
final_output: bool = False,
|
||||
final_output_type: str | None = None,
|
||||
is_comprehension: bool = False,
|
||||
):
|
||||
return types.SimpleNamespace(
|
||||
stage_id=stage_id,
|
||||
stage_type="llm",
|
||||
runtime_cfg={},
|
||||
prompt_expand_func=None,
|
||||
final_output=final_output,
|
||||
final_output_type=final_output_type,
|
||||
default_sampling_params=types.SimpleNamespace(name=f"sp-{stage_id}-{replica_id}"),
|
||||
custom_process_input_func=None,
|
||||
engine_input_source=[] if stage_id == 0 else [stage_id - 1],
|
||||
engine_output_type="token_ids",
|
||||
replica_id=replica_id,
|
||||
is_comprehension=is_comprehension,
|
||||
)
|
||||
|
||||
|
||||
def _make_diffusion_metadata(stage_id: int, *, replica_id: int = 0, final_output_type: str = "image"):
|
||||
return types.SimpleNamespace(
|
||||
stage_id=stage_id,
|
||||
stage_type="diffusion",
|
||||
runtime_cfg={"devices": str(replica_id)},
|
||||
prompt_expand_func=None,
|
||||
final_output=True,
|
||||
final_output_type=final_output_type,
|
||||
default_sampling_params=types.SimpleNamespace(name=f"dsp-{stage_id}-{replica_id}"),
|
||||
custom_process_input_func=None,
|
||||
engine_input_source=[],
|
||||
cfg_kv_collect_func=None,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
|
||||
|
||||
def _make_llm_plan(
|
||||
stage_idx: int,
|
||||
*,
|
||||
configured_stage_id: int,
|
||||
vllm_config: object,
|
||||
num_replicas: int = 1,
|
||||
final_output: bool = False,
|
||||
final_output_type: str | None = None,
|
||||
is_comprehension: bool = False,
|
||||
):
|
||||
replicas: list[ReplicaInitPlan] = []
|
||||
for replica_id in range(num_replicas):
|
||||
stage_cfg = types.SimpleNamespace(
|
||||
stage_id=configured_stage_id,
|
||||
stage_type="llm",
|
||||
runtime=types.SimpleNamespace(devices=str(replica_id)),
|
||||
engine_args={},
|
||||
)
|
||||
replicas.append(
|
||||
ReplicaInitPlan(
|
||||
replica_id=replica_id,
|
||||
num_replicas=num_replicas,
|
||||
launch_mode="local",
|
||||
stage_cfg=stage_cfg,
|
||||
metadata=_make_llm_metadata(
|
||||
configured_stage_id,
|
||||
replica_id=replica_id,
|
||||
final_output=final_output,
|
||||
final_output_type=final_output_type,
|
||||
is_comprehension=is_comprehension and replica_id == 0,
|
||||
),
|
||||
stage_connector_spec={},
|
||||
omni_kv_connector=(None, None, None),
|
||||
stage_vllm_config=vllm_config,
|
||||
executor_class=object,
|
||||
)
|
||||
)
|
||||
return LogicalStageInitPlan(
|
||||
stage_idx=stage_idx,
|
||||
configured_stage_id=configured_stage_id,
|
||||
replicas=replicas,
|
||||
)
|
||||
|
||||
|
||||
def _make_diffusion_plan(
|
||||
stage_idx: int,
|
||||
*,
|
||||
configured_stage_id: int,
|
||||
num_replicas: int = 1,
|
||||
):
|
||||
replicas: list[ReplicaInitPlan] = []
|
||||
for replica_id in range(num_replicas):
|
||||
stage_cfg = types.SimpleNamespace(
|
||||
stage_id=configured_stage_id,
|
||||
stage_type="diffusion",
|
||||
runtime=types.SimpleNamespace(devices=str(replica_id)),
|
||||
engine_args={},
|
||||
)
|
||||
replicas.append(
|
||||
ReplicaInitPlan(
|
||||
replica_id=replica_id,
|
||||
num_replicas=num_replicas,
|
||||
launch_mode="local",
|
||||
stage_cfg=stage_cfg,
|
||||
metadata=_make_diffusion_metadata(configured_stage_id, replica_id=replica_id),
|
||||
stage_connector_spec={},
|
||||
omni_kv_connector=(None, None, None),
|
||||
)
|
||||
)
|
||||
return LogicalStageInitPlan(
|
||||
stage_idx=stage_idx,
|
||||
configured_stage_id=configured_stage_id,
|
||||
replicas=replicas,
|
||||
)
|
||||
|
||||
|
||||
def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
|
||||
"""Regression test for forward references in make_async_mp_client."""
|
||||
import vllm_omni.engine.stage_engine_core_client as client_mod
|
||||
@@ -21,15 +144,279 @@ def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
|
||||
)
|
||||
|
||||
|
||||
def test_initialize_stages_restores_device_visibility_after_diffusion_init(monkeypatch):
|
||||
"""Regression test for stage device env leakage across stage init.
|
||||
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
|
||||
stage_cfg = types.SimpleNamespace(
|
||||
stage_id=0,
|
||||
stage_type="diffusion",
|
||||
engine_args={"parallel_config": {"tensor_parallel_size": 2}},
|
||||
runtime={"devices": "0,1,2,3", "num_replicas": 2},
|
||||
)
|
||||
|
||||
Diffusion init mutates process-level CUDA visibility. Ensure AsyncOmniEngine
|
||||
restores the previous value after diffusion stage setup.
|
||||
"""
|
||||
replicas_per_stage, replica_devices_map = compute_replica_layout([stage_cfg])
|
||||
|
||||
assert replicas_per_stage == [2]
|
||||
assert replica_devices_map == {0: ["0,1", "2,3"]}
|
||||
|
||||
|
||||
def test_collect_initialized_clients_for_cleanup_deduplicates_clients():
|
||||
shared = types.SimpleNamespace(name="shared")
|
||||
extra = types.SimpleNamespace(name="extra")
|
||||
|
||||
cleanup_clients = AsyncOmniEngine._collect_initialized_clients_for_cleanup(
|
||||
stage_pools=[types.SimpleNamespace(clients=[shared, None])],
|
||||
initialized_clients_by_stage={0: [shared], 1: [extra]},
|
||||
)
|
||||
|
||||
assert cleanup_clients == [shared, extra]
|
||||
|
||||
|
||||
def test_initialize_stages_rejects_non_diffusion_replicas_in_single_stage_mode():
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.single_stage_mode = True
|
||||
engine.stage_configs = [types.SimpleNamespace(stage_id=0, runtime={"num_replicas": 2})]
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="single_stage_mode only supports num_replicas > 1 for diffusion stages",
|
||||
):
|
||||
engine._validate_single_stage_mode_replica_constraints()
|
||||
|
||||
|
||||
def test_initialize_diffusion_replica_restores_device_visibility_after_local_init(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.model = "dummy-model"
|
||||
engine.num_stages = 1
|
||||
engine.diffusion_batch_size = 1
|
||||
engine.single_stage_mode = False
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = []
|
||||
|
||||
plan = _make_diffusion_plan(0, configured_stage_id=0).replicas[0]
|
||||
|
||||
env_var = current_omni_platform.device_control_env_var
|
||||
old_env = os.environ.get(env_var)
|
||||
os.environ[env_var] = "0,1"
|
||||
|
||||
def _fake_setup_stage_devices(_stage_id, _runtime_cfg):
|
||||
current_omni_platform.set_device_control_env_var("1")
|
||||
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices)
|
||||
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: types.SimpleNamespace())
|
||||
|
||||
try:
|
||||
engine._initialize_diffusion_replica(plan, stage_init_timeout=1, stage_launch_lock=threading.Lock())
|
||||
assert os.environ.get(env_var) == "0,1"
|
||||
finally:
|
||||
if old_env is None:
|
||||
os.environ.pop(env_var, None)
|
||||
else:
|
||||
os.environ[env_var] = old_env
|
||||
|
||||
|
||||
def test_initialize_diffusion_replica_passes_stage_init_timeout_and_inline_flag(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.model = "dummy-model"
|
||||
engine.num_stages = 1
|
||||
engine.diffusion_batch_size = 4
|
||||
engine.single_stage_mode = False
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = []
|
||||
|
||||
plan = _make_diffusion_plan(0, configured_stage_id=0).replicas[0]
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
|
||||
|
||||
def _capture_initialize_diffusion_stage(
|
||||
stage_id, _model, _stage_cfg, _metadata, *, stage_init_timeout, batch_size, use_inline
|
||||
):
|
||||
captured["stage_id"] = stage_id
|
||||
captured["stage_init_timeout"] = stage_init_timeout
|
||||
captured["batch_size"] = batch_size
|
||||
captured["use_inline"] = use_inline
|
||||
return types.SimpleNamespace()
|
||||
|
||||
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", _capture_initialize_diffusion_stage)
|
||||
|
||||
engine._initialize_diffusion_replica(plan, stage_init_timeout=302, stage_launch_lock=threading.Lock())
|
||||
|
||||
assert captured == {
|
||||
"stage_id": 0,
|
||||
"stage_init_timeout": 302,
|
||||
"batch_size": 4,
|
||||
"use_inline": True,
|
||||
}
|
||||
|
||||
|
||||
def test_initialize_stages_exposes_logical_stage_views_and_builds_top_level_input_processor(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.model = "dummy-model"
|
||||
engine.config_path = "dummy-config"
|
||||
engine.num_stages = 2
|
||||
engine.async_chunk = False
|
||||
engine.diffusion_batch_size = 1
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = [types.SimpleNamespace(), types.SimpleNamespace()]
|
||||
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
cfg1 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
stage_plans = [
|
||||
_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2, is_comprehension=True),
|
||||
_make_llm_plan(1, configured_stage_id=1, vllm_config=cfg1, final_output=True),
|
||||
]
|
||||
|
||||
stage0_client_r0 = types.SimpleNamespace(
|
||||
stage_type="llm",
|
||||
is_comprehension=True,
|
||||
final_output=False,
|
||||
final_output_type=None,
|
||||
default_sampling_params=types.SimpleNamespace(name="sp0"),
|
||||
)
|
||||
stage0_client_r1 = types.SimpleNamespace(
|
||||
stage_type="llm",
|
||||
is_comprehension=False,
|
||||
final_output=False,
|
||||
final_output_type=None,
|
||||
default_sampling_params=types.SimpleNamespace(name="sp0r1"),
|
||||
)
|
||||
stage1_client_r0 = types.SimpleNamespace(
|
||||
stage_type="llm",
|
||||
is_comprehension=False,
|
||||
final_output=True,
|
||||
final_output_type=None,
|
||||
default_sampling_params=types.SimpleNamespace(name="sp1"),
|
||||
)
|
||||
initialized_clients = {
|
||||
0: [stage0_client_r0, stage0_client_r1],
|
||||
1: [stage1_client_r0],
|
||||
}
|
||||
|
||||
stage0_output_processor = object()
|
||||
stage1_output_processor = object()
|
||||
top_level_input_processor = object()
|
||||
|
||||
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
|
||||
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2, 1], {}))
|
||||
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
|
||||
monkeypatch.setattr(engine, "_initialize_stage_replicas", lambda *_: initialized_clients)
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"build_llm_stage_output_processor",
|
||||
lambda plan, _cfg: stage0_output_processor if plan.stage_idx == 0 else stage1_output_processor,
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "build_stage0_input_processor", lambda _cfg: top_level_input_processor)
|
||||
|
||||
engine._initialize_stages(stage_init_timeout=1)
|
||||
|
||||
assert len(engine.stage_pools) == 2
|
||||
assert engine.input_processor is top_level_input_processor
|
||||
assert engine.stage_clients == [stage0_client_r0, stage1_client_r0]
|
||||
assert engine.stage_vllm_configs == [cfg0, cfg1]
|
||||
assert engine.output_processors == [stage0_output_processor, stage1_output_processor]
|
||||
assert engine.default_sampling_params_list == [
|
||||
stage0_client_r0.default_sampling_params,
|
||||
stage1_client_r0.default_sampling_params,
|
||||
]
|
||||
assert engine.stage_metadata == [
|
||||
{"final_output": False, "final_output_type": None, "stage_type": "llm"},
|
||||
{"final_output": True, "final_output_type": None, "stage_type": "llm"},
|
||||
]
|
||||
|
||||
|
||||
def test_build_logical_stage_init_plans_applies_replica_device_splits(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.model = "dummy-model"
|
||||
engine.async_chunk = False
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine.stage_configs = [
|
||||
types.SimpleNamespace(stage_id=0, stage_type="llm", engine_args={}, runtime=types.SimpleNamespace(devices="0")),
|
||||
types.SimpleNamespace(
|
||||
stage_id=1, stage_type="llm", engine_args={}, runtime=types.SimpleNamespace(devices="1,2,3")
|
||||
),
|
||||
]
|
||||
|
||||
metadata_by_stage = {
|
||||
0: _make_llm_metadata(0),
|
||||
1: _make_llm_metadata(1),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"extract_stage_metadata",
|
||||
lambda cfg: types.SimpleNamespace(**metadata_by_stage[cfg.stage_id].__dict__),
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "get_stage_connector_spec", lambda **_: {})
|
||||
monkeypatch.setattr(engine_mod, "resolve_omni_kv_config_for_stage", lambda *_: (None, None, None))
|
||||
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"build_vllm_config",
|
||||
lambda stage_cfg, *_args, **_kwargs: (types.SimpleNamespace(tag=f"cfg-{stage_cfg.stage_id}"), object),
|
||||
)
|
||||
|
||||
stage_plans, prompt_expand_func = engine._build_logical_stage_init_plans(
|
||||
omni_transfer_config=None,
|
||||
replicas_per_stage=[1, 3],
|
||||
replica_devices_map={1: ["1", "2", "3"]},
|
||||
)
|
||||
|
||||
assert prompt_expand_func is None
|
||||
assert [plan.configured_stage_id for plan in stage_plans] == [0, 1]
|
||||
assert [replica.stage_cfg.runtime.devices for replica in stage_plans[1].replicas] == ["1", "2", "3"]
|
||||
assert [replica.replica_id for replica in stage_plans[1].replicas] == [0, 1, 2]
|
||||
assert all(replica.num_replicas == 3 for replica in stage_plans[1].replicas)
|
||||
|
||||
|
||||
def test_initialize_stage_replicas_collects_results_by_stage_and_replica_id(monkeypatch):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
cfg1 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
stage_plans = [
|
||||
_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2),
|
||||
_make_llm_plan(1, configured_stage_id=1, vllm_config=cfg1, num_replicas=2),
|
||||
]
|
||||
|
||||
clients = {
|
||||
(0, 0): types.SimpleNamespace(name="stage0-replica0"),
|
||||
(0, 1): types.SimpleNamespace(name="stage0-replica1"),
|
||||
(1, 0): types.SimpleNamespace(name="stage1-replica0"),
|
||||
(1, 1): types.SimpleNamespace(name="stage1-replica1"),
|
||||
}
|
||||
|
||||
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
|
||||
time.sleep(0.02 * (3 - plan.metadata.stage_id - plan.replica_id))
|
||||
return clients[(plan.metadata.stage_id, plan.replica_id)]
|
||||
|
||||
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
|
||||
|
||||
initialized_clients = engine._initialize_stage_replicas(stage_plans, stage_init_timeout=123)
|
||||
|
||||
assert initialized_clients == {
|
||||
0: [clients[(0, 0)], clients[(0, 1)]],
|
||||
1: [clients[(1, 0)], clients[(1, 1)]],
|
||||
}
|
||||
|
||||
|
||||
def test_initialize_stages_cleans_up_successful_replicas_after_partial_multi_replica_failure(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.model = "dummy-model"
|
||||
engine.config_path = "dummy-config"
|
||||
@@ -39,158 +426,111 @@ def test_initialize_stages_restores_device_visibility_after_diffusion_init(monke
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = [types.SimpleNamespace(stage_id=0, stage_type="diffusion")]
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
|
||||
env_var = current_omni_platform.device_control_env_var
|
||||
old_env = os.environ.get(env_var)
|
||||
os.environ[env_var] = "0,1"
|
||||
|
||||
diffusion_client = types.SimpleNamespace(is_comprehension=False)
|
||||
|
||||
metadata = types.SimpleNamespace(
|
||||
stage_id=0,
|
||||
stage_type="diffusion",
|
||||
runtime_cfg={"devices": "1"},
|
||||
prompt_expand_func=None,
|
||||
)
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
stage_plans = [_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2)]
|
||||
initialized_client = types.SimpleNamespace(shutdown=lambda: None)
|
||||
|
||||
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
|
||||
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "extract_stage_metadata", lambda _cfg: metadata)
|
||||
monkeypatch.setattr(engine_mod, "get_stage_connector_spec", lambda **_: {})
|
||||
monkeypatch.setattr(engine_mod, "resolve_omni_kv_config_for_stage", lambda *_: (None, None, None))
|
||||
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2], {}))
|
||||
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
|
||||
|
||||
def _fake_setup_stage_devices(_stage_id, _runtime_cfg):
|
||||
# Simulate diffusion setup mutating process-global visibility.
|
||||
current_omni_platform.set_device_control_env_var("1")
|
||||
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
|
||||
if plan.replica_id == 0:
|
||||
return initialized_client
|
||||
time.sleep(0.05)
|
||||
raise RuntimeError("replica launch failed")
|
||||
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices)
|
||||
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: diffusion_client)
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"finalize_initialized_stages",
|
||||
lambda stage_clients, _input_processor: (
|
||||
stage_clients,
|
||||
[types.SimpleNamespace()],
|
||||
[{"final_output_type": "image"}],
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
|
||||
|
||||
try:
|
||||
captured_cleanup: list[list[object]] = []
|
||||
|
||||
def _capture_shutdown(clients):
|
||||
captured_cleanup.append(list(clients))
|
||||
|
||||
monkeypatch.setattr(engine, "_shutdown_initialized_clients", _capture_shutdown)
|
||||
|
||||
with pytest.raises(RuntimeError, match="replica launch failed"):
|
||||
engine._initialize_stages(stage_init_timeout=1)
|
||||
assert os.environ.get(env_var) == "0,1"
|
||||
finally:
|
||||
if old_env is None:
|
||||
os.environ.pop(env_var, None)
|
||||
else:
|
||||
os.environ[env_var] = old_env
|
||||
|
||||
assert captured_cleanup == [[initialized_client]]
|
||||
|
||||
|
||||
def test_initialize_stages_passes_stage_init_timeout_to_diffusion_handshake(monkeypatch):
|
||||
"""Regression test for stage_init_timeout passing to complete_diffusion_handshake
|
||||
in the diffusion stage path.
|
||||
"""
|
||||
import vllm_omni.diffusion.data as diffusion_data_mod
|
||||
import vllm_omni.diffusion.stage_diffusion_client as client_mod
|
||||
def test_initialize_stages_cleans_up_late_successful_replicas_after_early_multi_replica_failure(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.log_stats = False
|
||||
engine.model = "dummy-model"
|
||||
engine.config_path = "dummy-config"
|
||||
engine.num_stages = 2
|
||||
engine.num_stages = 1
|
||||
engine.async_chunk = False
|
||||
engine.diffusion_batch_size = 1
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = [types.SimpleNamespace(stage_id=0, stage_type="diffusion", engine_args={})]
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
|
||||
metadata = types.SimpleNamespace(
|
||||
stage_id=0,
|
||||
stage_type="diffusion",
|
||||
runtime_cfg={"devices": "0"},
|
||||
prompt_expand_func=None,
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
default_sampling_params=None,
|
||||
custom_process_input_func=None,
|
||||
engine_input_source=None,
|
||||
cfg_kv_collect_func=None,
|
||||
)
|
||||
|
||||
captured_timeout = None
|
||||
device_env_var = current_omni_platform.device_control_env_var
|
||||
prev_device_env = os.environ.get(device_env_var)
|
||||
os.environ[device_env_var] = "0"
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
stage_plans = [_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2)]
|
||||
initialized_client = types.SimpleNamespace(shutdown=lambda: None)
|
||||
replica1_started = threading.Event()
|
||||
|
||||
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
|
||||
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "extract_stage_metadata", lambda _cfg: metadata)
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"finalize_initialized_stages",
|
||||
lambda stage_clients, _input_processor: (
|
||||
stage_clients,
|
||||
[types.SimpleNamespace()],
|
||||
[{"final_output_type": "image"}],
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
diffusion_data_mod.OmniDiffusionConfig,
|
||||
"from_kwargs",
|
||||
classmethod(lambda cls, **kwargs: types.SimpleNamespace(parallel_config=types.SimpleNamespace(world_size=1))),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
client_mod,
|
||||
"spawn_diffusion_proc",
|
||||
lambda model, od_cfg: (object(), "ipc://handshake", "ipc://request", "ipc://response"),
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2], {}))
|
||||
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
|
||||
|
||||
def _capture_handshake_timeout(proc, handshake_address, handshake_timeout):
|
||||
nonlocal captured_timeout
|
||||
captured_timeout = handshake_timeout
|
||||
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
|
||||
if plan.replica_id == 0:
|
||||
assert replica1_started.wait(timeout=1)
|
||||
raise RuntimeError("replica launch failed")
|
||||
replica1_started.set()
|
||||
time.sleep(0.05)
|
||||
return initialized_client
|
||||
|
||||
monkeypatch.setattr(client_mod, "complete_diffusion_handshake", _capture_handshake_timeout)
|
||||
monkeypatch.setattr(
|
||||
client_mod.zmq,
|
||||
"Context",
|
||||
lambda: types.SimpleNamespace(socket=lambda _: types.SimpleNamespace(connect=lambda _: None)),
|
||||
)
|
||||
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
|
||||
|
||||
try:
|
||||
engine._initialize_stages(stage_init_timeout=302)
|
||||
finally:
|
||||
if prev_device_env is None:
|
||||
os.environ.pop(device_env_var, None)
|
||||
else:
|
||||
os.environ[device_env_var] = prev_device_env
|
||||
captured_cleanup: list[list[object]] = []
|
||||
|
||||
assert captured_timeout == 302
|
||||
def _capture_shutdown(clients):
|
||||
captured_cleanup.append(list(clients))
|
||||
|
||||
monkeypatch.setattr(engine, "_shutdown_initialized_clients", _capture_shutdown)
|
||||
|
||||
with pytest.raises(RuntimeError, match="replica launch failed"):
|
||||
engine._initialize_stages(stage_init_timeout=1)
|
||||
|
||||
assert captured_cleanup == [[initialized_client]]
|
||||
|
||||
|
||||
def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(monkeypatch):
|
||||
"""Regression test for stage_init_timeout reaching complete_stage_handshake
|
||||
in the LLM stage path.
|
||||
"""
|
||||
def test_initialize_llm_replica_passes_stage_init_timeout_to_complete_stage_handshake(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.log_stats = False
|
||||
engine.model = "dummy-model"
|
||||
engine.single_stage_mode = False
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = []
|
||||
|
||||
metadata = types.SimpleNamespace(stage_id=0, runtime_cfg={"devices": "0"})
|
||||
fake_vllm_config = types.SimpleNamespace()
|
||||
fake_addresses = types.SimpleNamespace()
|
||||
fake_addresses = types.SimpleNamespace(inputs=["in"], outputs=["out"], frontend_stats_publish_address=None)
|
||||
fake_proc = types.SimpleNamespace()
|
||||
captured_timeout: int | None = None
|
||||
|
||||
captured_timeout = None
|
||||
plan = ReplicaInitPlan(
|
||||
replica_id=0,
|
||||
num_replicas=1,
|
||||
launch_mode="local",
|
||||
stage_cfg=types.SimpleNamespace(engine_args={}, runtime=types.SimpleNamespace(devices="0")),
|
||||
metadata=types.SimpleNamespace(stage_id=0, runtime_cfg={"devices": "0"}),
|
||||
stage_connector_spec={},
|
||||
omni_kv_connector=(None, None, None),
|
||||
stage_vllm_config=fake_vllm_config,
|
||||
executor_class=object,
|
||||
)
|
||||
|
||||
device_env_var = current_omni_platform.device_control_env_var
|
||||
prev_device_env = os.environ.get(device_env_var)
|
||||
@@ -198,28 +538,22 @@ def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(
|
||||
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
|
||||
monkeypatch.setattr(engine_mod, "build_vllm_config", lambda *_, **__: (fake_vllm_config, object))
|
||||
monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: [])
|
||||
monkeypatch.setattr(
|
||||
engine_mod,
|
||||
"spawn_stage_core",
|
||||
lambda **_: (fake_addresses, fake_proc, "ipc://handshake"),
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "spawn_stage_core", lambda **_: (fake_addresses, fake_proc, "ipc://handshake"))
|
||||
|
||||
def _capture_stage_timeout(_proc, _handshake_addr, _addresses, _vllm_cfg, handshake_timeout):
|
||||
nonlocal captured_timeout
|
||||
captured_timeout = handshake_timeout
|
||||
|
||||
monkeypatch.setattr(engine_mod, "complete_stage_handshake", _capture_stage_timeout)
|
||||
monkeypatch.setattr(
|
||||
engine_mod.StageEngineCoreClientBase,
|
||||
"make_async_mp_client",
|
||||
staticmethod(lambda **_: types.SimpleNamespace(shutdown=lambda: None)),
|
||||
)
|
||||
|
||||
try:
|
||||
engine._launch_llm_stage(
|
||||
stage_cfg=types.SimpleNamespace(engine_args={}),
|
||||
metadata=metadata,
|
||||
stage_connector_spec={},
|
||||
stage_init_timeout=302,
|
||||
llm_stage_launch_lock=threading.Lock(),
|
||||
)
|
||||
engine._initialize_llm_replica(plan, 302, threading.Lock())
|
||||
finally:
|
||||
if prev_device_env is None:
|
||||
os.environ.pop(device_env_var, None)
|
||||
@@ -229,155 +563,27 @@ def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(
|
||||
assert captured_timeout == 302
|
||||
|
||||
|
||||
def test_launch_llm_stage_releases_launch_lock_before_complete_stage_handshake(monkeypatch):
|
||||
"""Regression test for parallel LLM stage startup during handshake wait."""
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.log_stats = False
|
||||
engine.model = "dummy-model"
|
||||
engine.single_stage_mode = False
|
||||
engine._omni_master_server = None
|
||||
engine.stage_configs = []
|
||||
|
||||
fake_vllm_config = types.SimpleNamespace()
|
||||
fake_addresses = types.SimpleNamespace()
|
||||
shared_launch_lock = threading.Lock()
|
||||
counter_lock = threading.Lock()
|
||||
first_handshake_started = threading.Event()
|
||||
second_stage_spawned = threading.Event()
|
||||
allow_first_handshake_to_finish = threading.Event()
|
||||
launch_errors: list[BaseException] = []
|
||||
spawn_count = 0
|
||||
|
||||
device_env_var = current_omni_platform.device_control_env_var
|
||||
prev_device_env = os.environ.get(device_env_var)
|
||||
os.environ[device_env_var] = "0"
|
||||
|
||||
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
|
||||
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
|
||||
monkeypatch.setattr(engine_mod, "build_vllm_config", lambda *_, **__: (fake_vllm_config, object))
|
||||
monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: [])
|
||||
|
||||
def _spawn_stage_core(**_):
|
||||
nonlocal spawn_count
|
||||
with counter_lock:
|
||||
spawn_count += 1
|
||||
call_idx = spawn_count
|
||||
if call_idx == 2:
|
||||
second_stage_spawned.set()
|
||||
return fake_addresses, types.SimpleNamespace(), f"ipc://handshake-{call_idx}"
|
||||
|
||||
def _complete_stage_handshake(_proc, handshake_address, _addresses, _vllm_cfg, _timeout):
|
||||
if handshake_address == "ipc://handshake-1":
|
||||
first_handshake_started.set()
|
||||
assert second_stage_spawned.wait(timeout=1), (
|
||||
"second stage did not reach spawn_stage_core while first stage waited in handshake"
|
||||
)
|
||||
assert allow_first_handshake_to_finish.wait(timeout=1), (
|
||||
"second stage did not enter handshake while first stage was still waiting"
|
||||
)
|
||||
else:
|
||||
allow_first_handshake_to_finish.set()
|
||||
|
||||
monkeypatch.setattr(engine_mod, "spawn_stage_core", _spawn_stage_core)
|
||||
monkeypatch.setattr(engine_mod, "complete_stage_handshake", _complete_stage_handshake)
|
||||
|
||||
def _launch_stage(stage_id: int) -> None:
|
||||
metadata = types.SimpleNamespace(stage_id=stage_id, runtime_cfg={"devices": str(stage_id)})
|
||||
try:
|
||||
engine._launch_llm_stage(
|
||||
stage_cfg=types.SimpleNamespace(engine_args={}),
|
||||
metadata=metadata,
|
||||
stage_connector_spec={},
|
||||
stage_init_timeout=302,
|
||||
llm_stage_launch_lock=shared_launch_lock,
|
||||
)
|
||||
except BaseException as exc: # pragma: no cover - surfaced through assertion below
|
||||
launch_errors.append(exc)
|
||||
|
||||
try:
|
||||
first_thread = threading.Thread(target=_launch_stage, args=(0,))
|
||||
first_thread.start()
|
||||
assert first_handshake_started.wait(timeout=1), "first stage never entered handshake"
|
||||
|
||||
second_thread = threading.Thread(target=_launch_stage, args=(1,))
|
||||
second_thread.start()
|
||||
|
||||
first_thread.join(timeout=3)
|
||||
second_thread.join(timeout=3)
|
||||
finally:
|
||||
if prev_device_env is None:
|
||||
os.environ.pop(device_env_var, None)
|
||||
else:
|
||||
os.environ[device_env_var] = prev_device_env
|
||||
|
||||
assert not first_thread.is_alive()
|
||||
assert not second_thread.is_alive()
|
||||
assert second_stage_spawned.is_set()
|
||||
assert not launch_errors
|
||||
|
||||
|
||||
def test_attach_llm_stage_uses_omni_input_preprocessor(monkeypatch):
|
||||
"""Regression test for GLM-Image t2i preprocessing path.
|
||||
|
||||
Stage-0 InputProcessor must use OmniInputPreprocessor so text prompts with
|
||||
mm_processor_kwargs go through multimodal preprocessing.
|
||||
"""
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
class DummyStageEngineCoreClient:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
def shutdown(self):
|
||||
return None
|
||||
|
||||
class DummyOutputProcessor:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
def test_build_stage0_input_processor_uses_omni_input_preprocessor(monkeypatch):
|
||||
import vllm_omni.engine.stage_init_utils as init_mod
|
||||
|
||||
class DummyInputProcessor:
|
||||
def __init__(self, vllm_config):
|
||||
self.vllm_config = vllm_config
|
||||
self.renderer = object()
|
||||
self.input_preprocessor = object()
|
||||
self.input_preprocessor = None
|
||||
|
||||
class DummyOmniInputPreprocessor:
|
||||
def __init__(self, vllm_config, renderer=None):
|
||||
self.vllm_config = vllm_config
|
||||
self.renderer = renderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
engine_mod.StageEngineCoreClientBase,
|
||||
"make_async_mp_client",
|
||||
staticmethod(lambda **kwargs: DummyStageEngineCoreClient(**kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(engine_mod, "MultimodalOutputProcessor", DummyOutputProcessor)
|
||||
monkeypatch.setattr(engine_mod, "InputProcessor", DummyInputProcessor)
|
||||
monkeypatch.setattr(engine_mod, "OmniInputPreprocessor", DummyOmniInputPreprocessor)
|
||||
monkeypatch.setattr(init_mod, "InputProcessor", DummyInputProcessor)
|
||||
monkeypatch.setattr(init_mod, "OmniInputPreprocessor", DummyOmniInputPreprocessor)
|
||||
|
||||
started = types.SimpleNamespace(
|
||||
stage_id=0,
|
||||
metadata=types.SimpleNamespace(stage_id=0, engine_output_type="token_ids"),
|
||||
vllm_config=types.SimpleNamespace(model_config=types.SimpleNamespace(skip_tokenizer_init=True)),
|
||||
executor_class=object,
|
||||
engine_manager=object(),
|
||||
coordinator=object(),
|
||||
proc=None,
|
||||
addresses=types.SimpleNamespace(
|
||||
inputs=["inproc://input"],
|
||||
outputs=["inproc://output"],
|
||||
frontend_stats_publish_address=None,
|
||||
),
|
||||
input_processor = build_stage0_input_processor(
|
||||
types.SimpleNamespace(model_config=types.SimpleNamespace(try_get_generation_config=lambda: {}))
|
||||
)
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
|
||||
_stage_client, _out_proc, _vllm_cfg, input_processor = engine._attach_llm_stage(started)
|
||||
|
||||
assert input_processor is not None
|
||||
assert isinstance(input_processor.input_preprocessor, DummyOmniInputPreprocessor)
|
||||
assert input_processor.input_preprocessor.renderer is input_processor.renderer
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -25,18 +24,6 @@ def test_register_companion_and_cleanup():
|
||||
assert tracker.get_companion_request_ids("req1") == {}
|
||||
|
||||
|
||||
def test_attach_cfg_request_ids_clones_diffusion_params():
|
||||
tracker = CfgCompanionTracker()
|
||||
tracker.register_companion("req1", "cfg_text", "req1__cfg_text")
|
||||
|
||||
params = OmniDiffusionSamplingParams()
|
||||
updated = tracker.attach_cfg_request_ids("req1", params)
|
||||
|
||||
assert updated is not params
|
||||
assert params.cfg_kv_request_ids is None
|
||||
assert updated.cfg_kv_request_ids == {"cfg_text": "req1__cfg_text"}
|
||||
|
||||
|
||||
def test_abort_parent_expands_to_companions_and_cleans_up_deferred_parent():
|
||||
tracker = CfgCompanionTracker()
|
||||
tracker.register_companion("req1", "cfg_text", "req1__cfg_text")
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -16,8 +17,9 @@ from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.core_client import AsyncMPClient
|
||||
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -73,7 +75,7 @@ class FakeStageClient:
|
||||
def set_engine_outputs(self, outputs) -> None:
|
||||
return None
|
||||
|
||||
def process_engine_inputs(self, stage_list, prompt=None, streaming_context=None):
|
||||
def process_engine_inputs(self, source_outputs, prompt=None, streaming_context=None):
|
||||
return list(self.next_inputs)
|
||||
|
||||
async def abort_requests_async(self, request_ids: list[str]) -> None:
|
||||
@@ -90,12 +92,33 @@ class FakeStageClient:
|
||||
self._diffusion_outputs.put_nowait(output)
|
||||
|
||||
|
||||
class FakeCollectiveRpcStageClient(FakeStageClient):
|
||||
def __init__(self, *args, rpc_result: Any = None, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.rpc_result = rpc_result
|
||||
self.collective_rpc_calls: list[tuple[str, float | None, tuple[Any, ...], dict[str, Any]]] = []
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
timeout: float | None = None,
|
||||
args: tuple[Any, ...] = (),
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
normalized_kwargs = dict(kwargs or {})
|
||||
self.collective_rpc_calls.append((method, timeout, args, normalized_kwargs))
|
||||
return self.rpc_result
|
||||
|
||||
|
||||
class FakeOutputProcessor:
|
||||
def __init__(self, *, request_outputs: list[object] | None = None) -> None:
|
||||
self.request_outputs = list(request_outputs or [])
|
||||
self.add_request_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
|
||||
self.abort_calls: list[list[str]] = []
|
||||
|
||||
def add_request(self, *_args, **_kwargs) -> None:
|
||||
def add_request(self, *args, **kwargs) -> None:
|
||||
self.add_request_calls.append((args, kwargs))
|
||||
return None
|
||||
|
||||
def process_outputs(self, *_args, **_kwargs):
|
||||
@@ -185,17 +208,60 @@ def _build_request_output(
|
||||
)
|
||||
|
||||
|
||||
def _build_stage_pools(
|
||||
stage_clients: list[list[FakeStageClient]],
|
||||
*,
|
||||
output_processors: list[FakeOutputProcessor] | None = None,
|
||||
stage_vllm_configs: list[object] | None = None,
|
||||
) -> list[StagePool]:
|
||||
"""Build StagePool list from per-stage replica lists.
|
||||
|
||||
``stage_clients[i]`` is the list of FakeStageClient replicas for stage i.
|
||||
"""
|
||||
num_stages = len(stage_clients)
|
||||
if output_processors is None:
|
||||
output_processors = [FakeOutputProcessor() for _ in stage_clients]
|
||||
if stage_vllm_configs is None:
|
||||
stage_vllm_configs = [SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)) for _ in stage_clients]
|
||||
|
||||
pools: list[StagePool] = []
|
||||
for stage_id in range(num_stages):
|
||||
clients = stage_clients[stage_id]
|
||||
if clients[0].stage_type == "diffusion":
|
||||
pools.append(StagePool(stage_id, clients[0]))
|
||||
else:
|
||||
pools.append(
|
||||
StagePool(
|
||||
stage_id,
|
||||
clients,
|
||||
output_processor=output_processors[stage_id],
|
||||
stage_vllm_config=stage_vllm_configs[stage_id],
|
||||
)
|
||||
)
|
||||
return pools
|
||||
|
||||
|
||||
def _build_harness(
|
||||
stage_clients: list[object],
|
||||
*,
|
||||
output_processors: list[object] | None = None,
|
||||
stage_vllm_configs: list[object] | None = None,
|
||||
async_chunk: bool = False,
|
||||
stage_pools: list[StagePool] | None = None,
|
||||
) -> OrchestratorFixture:
|
||||
if output_processors is None:
|
||||
output_processors = [FakeOutputProcessor() for _ in stage_clients]
|
||||
if stage_vllm_configs is None:
|
||||
stage_vllm_configs = [SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)) for _ in stage_clients]
|
||||
"""Build an Orchestrator test harness.
|
||||
|
||||
Accepts either pre-built ``stage_pools`` or flat lists of single-replica
|
||||
clients/processors.
|
||||
"""
|
||||
if stage_pools is None:
|
||||
# Wrap flat lists into per-stage single-replica lists.
|
||||
nested_clients = [[c] for c in stage_clients]
|
||||
stage_pools = _build_stage_pools(
|
||||
nested_clients,
|
||||
output_processors=output_processors,
|
||||
stage_vllm_configs=stage_vllm_configs,
|
||||
)
|
||||
|
||||
ready_future: concurrent.futures.Future[tuple[Orchestrator, janus.Queue, janus.Queue, janus.Queue]] = (
|
||||
concurrent.futures.Future()
|
||||
@@ -214,9 +280,7 @@ def _build_harness(
|
||||
request_async_queue=request_queue.async_q,
|
||||
output_async_queue=output_queue.async_q,
|
||||
rpc_async_queue=rpc_queue.async_q,
|
||||
stage_clients=stage_clients,
|
||||
output_processors=output_processors,
|
||||
stage_vllm_configs=stage_vllm_configs,
|
||||
stage_pools=stage_pools,
|
||||
async_chunk=async_chunk,
|
||||
)
|
||||
ready_future.set_result((orchestrator, request_queue, output_queue, rpc_queue))
|
||||
@@ -283,6 +347,18 @@ async def _get_output_message(orchestrator_fixture: OrchestratorFixture, *, time
|
||||
return msg
|
||||
|
||||
|
||||
async def _get_rpc_message(orchestrator_fixture: OrchestratorFixture, *, timeout: float = 2.0) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
rpc_sync_q = orchestrator_fixture.queues[2].sync_q
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
raise AssertionError("Timed out waiting for orchestrator rpc output")
|
||||
try:
|
||||
return rpc_sync_q.get_nowait()
|
||||
except queue.Empty:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
async def _enqueue_add_request(
|
||||
orchestrator_fixture: OrchestratorFixture,
|
||||
*,
|
||||
@@ -387,6 +463,11 @@ def orchestrator_factory():
|
||||
q.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Existing single-replica tests (adapted to StagePool interface)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_two_stage_llm(orchestrator_factory) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
@@ -600,12 +681,425 @@ async def test_run_abort(orchestrator_factory) -> None:
|
||||
await _wait_for(lambda: len(stages[0].add_request_calls) == 1)
|
||||
|
||||
await _enqueue_abort_request(orchestrator_fixture, ["req-abort"])
|
||||
await _wait_for(lambda: all(stage.abort_calls for stage in stages))
|
||||
await _wait_for(lambda: bool(stages[0].abort_calls))
|
||||
|
||||
for stage in stages:
|
||||
assert stage.abort_calls == [["req-abort"]]
|
||||
for proc in processors:
|
||||
assert proc.abort_calls == [["req-abort"]]
|
||||
assert stages[0].abort_calls == [["req-abort"]]
|
||||
assert stages[1].abort_calls == []
|
||||
assert "req-abort" not in orchestrator_fixture.orchestrator.request_states
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-replica tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_replica_round_robin_distribution(orchestrator_factory) -> None:
|
||||
"""Two replicas at stage-0, single replica at stage-1.
|
||||
|
||||
Send two requests — they should land on different stage-0 replicas
|
||||
(round-robin), then both forward to the single stage-1 replica.
|
||||
"""
|
||||
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(
|
||||
stage_type="llm",
|
||||
final_output=True,
|
||||
next_inputs=[{"prompt_token_ids": [7, 8]}],
|
||||
)
|
||||
|
||||
proc0 = FakeOutputProcessor(request_outputs=[_build_request_output("req-0", token_ids=[3], finished=True)])
|
||||
proc1 = FakeOutputProcessor(request_outputs=[_build_request_output("req-0", token_ids=[10], finished=True)])
|
||||
|
||||
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0_r0, stage0_r1], [stage1]],
|
||||
output_processors=[proc0, proc1],
|
||||
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
|
||||
)
|
||||
|
||||
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
|
||||
|
||||
try:
|
||||
# Request 0 → should land on replica 0 (RR starts at 0)
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-0",
|
||||
prompt=SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
|
||||
original_prompt={"prompt": "hello 0"},
|
||||
sampling_params_list=[_sampling_params(), _sampling_params()],
|
||||
final_stage_id=1,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
|
||||
assert len(stage0_r1.add_request_calls) == 0
|
||||
|
||||
# Request 1 → should land on replica 1 (RR advances)
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-1",
|
||||
prompt=SimpleNamespace(request_id="req-1", prompt_token_ids=[5, 6]),
|
||||
original_prompt={"prompt": "hello 1"},
|
||||
sampling_params_list=[_sampling_params(), _sampling_params()],
|
||||
final_stage_id=1,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 1)
|
||||
assert len(stage0_r0.add_request_calls) == 1 # unchanged
|
||||
|
||||
# Complete req-0 at stage-0 replica-0 → should forward to stage-1
|
||||
stage0_r0.push_engine_core_outputs(_engine_core_outputs("s0r0-raw", 1.0))
|
||||
await _wait_for(lambda: len(stage1.add_request_calls) == 1)
|
||||
assert stage1.add_request_calls[0][0].request_id == "req-0"
|
||||
|
||||
# Complete req-0 at stage-1 → final output
|
||||
proc1.request_outputs = [_build_request_output("req-0", token_ids=[10], finished=True)]
|
||||
stage1.push_engine_core_outputs(_engine_core_outputs("s1-raw", 2.0))
|
||||
output_msg = await _get_output_message(orchestrator_fixture)
|
||||
|
||||
assert output_msg["request_id"] == "req-0"
|
||||
assert output_msg["stage_id"] == 1
|
||||
assert output_msg["finished"] is True
|
||||
assert "req-0" not in orchestrator_fixture.orchestrator.request_states
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_replica_abort_broadcasts_to_all_replicas(orchestrator_factory) -> None:
|
||||
"""Abort must be sent to every replica across all stages."""
|
||||
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
|
||||
proc0 = FakeOutputProcessor()
|
||||
proc1 = FakeOutputProcessor()
|
||||
|
||||
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0_r0, stage0_r1], [stage1]],
|
||||
output_processors=[proc0, proc1],
|
||||
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
|
||||
)
|
||||
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
|
||||
|
||||
try:
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-abort-mr",
|
||||
prompt=SimpleNamespace(request_id="req-abort-mr", prompt_token_ids=[1]),
|
||||
original_prompt={"prompt": "cancel"},
|
||||
sampling_params_list=[_sampling_params(), _sampling_params()],
|
||||
final_stage_id=1,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
|
||||
|
||||
await _enqueue_abort_request(orchestrator_fixture, ["req-abort-mr"])
|
||||
await _wait_for(lambda: bool(stage0_r0.abort_calls))
|
||||
|
||||
assert stage0_r0.abort_calls == [["req-abort-mr"]]
|
||||
assert stage0_r1.abort_calls == []
|
||||
assert stage1.abort_calls == []
|
||||
assert "req-abort-mr" not in orchestrator_fixture.orchestrator.request_states
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_replica_shutdown_all_replicas(orchestrator_factory) -> None:
|
||||
"""Shutdown must shut down every replica across all stages."""
|
||||
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
|
||||
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0_r0, stage0_r1], [stage1]],
|
||||
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
|
||||
)
|
||||
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
|
||||
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
assert not orchestrator_fixture.thread.is_alive()
|
||||
for client in [stage0_r0, stage0_r1, stage1]:
|
||||
assert client.shutdown_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_submit_update_reuses_existing_binding() -> None:
|
||||
"""A request admitted to one replica must keep using that replica on updates."""
|
||||
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
pool = StagePool(
|
||||
0,
|
||||
[stage0_r0, stage0_r1],
|
||||
output_processor=FakeOutputProcessor(),
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
|
||||
req0_state = OrchestratorRequestState(
|
||||
request_id="req-0",
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
req1_state = OrchestratorRequestState(
|
||||
request_id="req-1",
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
|
||||
await pool.submit_initial("req-0", req0_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]))
|
||||
await pool.submit_update("req-0", req0_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[3]))
|
||||
await pool.submit_initial("req-1", req1_state, SimpleNamespace(request_id="req-1", prompt_token_ids=[4, 5]))
|
||||
await pool.submit_update("req-1", req1_state, SimpleNamespace(request_id="req-1", prompt_token_ids=[6]))
|
||||
|
||||
assert pool.get_bound_replica_id("req-0") == 0
|
||||
assert pool.get_bound_replica_id("req-1") == 1
|
||||
assert len(stage0_r0.add_request_calls) == 2
|
||||
assert len(stage0_r1.add_request_calls) == 2
|
||||
assert stage0_r0.add_request_calls[0][0].request_id == "req-0"
|
||||
assert stage0_r0.add_request_calls[1][0].request_id == "req-0"
|
||||
assert stage0_r1.add_request_calls[0][0].request_id == "req-1"
|
||||
assert stage0_r1.add_request_calls[1][0].request_id == "req-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_submit_update_refreshes_output_processor_state() -> None:
|
||||
output_processor = FakeOutputProcessor()
|
||||
|
||||
class AssertingStageClient(FakeStageClient):
|
||||
async def add_request_async(self, *args, **kwargs) -> None:
|
||||
if len(self.add_request_calls) == 1:
|
||||
prompts = [call_kwargs["prompt"] for _, call_kwargs in output_processor.add_request_calls]
|
||||
assert prompts == ["seg-1", "seg-2"]
|
||||
await super().add_request_async(*args, **kwargs)
|
||||
|
||||
stage0 = AssertingStageClient(stage_type="llm", final_output=False)
|
||||
pool = StagePool(
|
||||
0,
|
||||
[stage0],
|
||||
output_processor=output_processor,
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id="req-0",
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
|
||||
await pool.submit_initial(
|
||||
"req-0",
|
||||
req_state,
|
||||
SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
|
||||
prompt_text="seg-1",
|
||||
)
|
||||
await pool.submit_update(
|
||||
"req-0",
|
||||
req_state,
|
||||
SimpleNamespace(request_id="req-0", prompt_token_ids=[3], resumable=True),
|
||||
prompt_text="seg-2",
|
||||
)
|
||||
|
||||
assert len(output_processor.add_request_calls) == 2
|
||||
assert output_processor.add_request_calls[1][1]["prompt"] == "seg-2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> None:
|
||||
class RecordingPool:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, Any]] = []
|
||||
|
||||
async def submit_update(self, request_id, req_state, request, *, prompt_text=None) -> int:
|
||||
self.calls.append((request_id, prompt_text))
|
||||
return 0
|
||||
|
||||
pool = RecordingPool()
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.request_states = {
|
||||
"req-stream": OrchestratorRequestState(
|
||||
request_id="req-stream",
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
}
|
||||
orchestrator.stage_pools = [pool]
|
||||
|
||||
await orchestrator._handle_streaming_update(
|
||||
{
|
||||
"request_id": "req-stream",
|
||||
"prompt": SimpleNamespace(request_id="req-stream", prompt_token_ids=[1], resumable=True),
|
||||
"sampling_params_list": [_sampling_params()],
|
||||
"output_prompt_text": "segment-2",
|
||||
}
|
||||
)
|
||||
|
||||
assert pool.calls == [("req-stream", "segment-2")]
|
||||
assert orchestrator.request_states["req-stream"].streaming.enabled is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_submit_initial_rolls_back_output_processor_when_client_submit_fails() -> None:
|
||||
class FailingStageClient(FakeStageClient):
|
||||
async def add_request_async(self, *args, **_kwargs) -> None:
|
||||
raise RuntimeError("submit failed")
|
||||
|
||||
class TrackingOutputProcessor(FakeOutputProcessor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.added_request_ids: list[str] = []
|
||||
self.removed_request_ids: list[str] = []
|
||||
|
||||
def add_request(self, request, *_args, **_kwargs) -> None:
|
||||
self.added_request_ids.append(request.request_id)
|
||||
|
||||
def remove_request(self, request_id: str) -> None:
|
||||
self.removed_request_ids.append(request_id)
|
||||
|
||||
client = FailingStageClient(stage_type="llm", final_output=False)
|
||||
output_processor = TrackingOutputProcessor()
|
||||
pool = StagePool(
|
||||
0,
|
||||
[client],
|
||||
output_processor=output_processor,
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id="req-0",
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="submit failed"):
|
||||
await pool.submit_initial("req-0", req_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]))
|
||||
|
||||
assert output_processor.added_request_ids == ["req-0"]
|
||||
assert output_processor.removed_request_ids == ["req-0"]
|
||||
assert pool.get_bound_replica_id("req-0") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_pool_abort_requests_logs_when_binding_is_missing(caplog) -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
pool = StagePool(
|
||||
0,
|
||||
[stage0],
|
||||
output_processor=FakeOutputProcessor(),
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
|
||||
target_logger = logging.getLogger("vllm_omni.engine.stage_pool")
|
||||
target_logger.addHandler(caplog.handler)
|
||||
prev_level = target_logger.level
|
||||
target_logger.setLevel(logging.DEBUG)
|
||||
try:
|
||||
await pool.abort_requests(["missing-req"])
|
||||
finally:
|
||||
target_logger.removeHandler(caplog.handler)
|
||||
target_logger.setLevel(prev_level)
|
||||
|
||||
assert not stage0.abort_calls
|
||||
assert "abort: no binding for req=missing-req in stage-0" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collective_rpc_ignores_invalid_stage_ids(orchestrator_factory, caplog) -> None:
|
||||
stage0 = FakeCollectiveRpcStageClient(stage_type="llm", final_output=True, rpc_result={"stage": 0})
|
||||
stage1 = FakeCollectiveRpcStageClient(stage_type="llm", final_output=True, rpc_result={"stage": 1})
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0], [stage1]],
|
||||
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
|
||||
stage_vllm_configs=[
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
],
|
||||
)
|
||||
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
|
||||
|
||||
try:
|
||||
target_logger = logging.getLogger("vllm_omni.engine.orchestrator")
|
||||
target_logger.addHandler(caplog.handler)
|
||||
prev_level = target_logger.level
|
||||
target_logger.setLevel(logging.WARNING)
|
||||
try:
|
||||
orchestrator_fixture.request_sync_q.put_nowait(
|
||||
{
|
||||
"type": "collective_rpc",
|
||||
"rpc_id": "rpc-1",
|
||||
"method": "list_loras",
|
||||
"stage_ids": [99, 1],
|
||||
}
|
||||
)
|
||||
|
||||
msg = await _get_rpc_message(orchestrator_fixture)
|
||||
finally:
|
||||
target_logger.removeHandler(caplog.handler)
|
||||
target_logger.setLevel(prev_level)
|
||||
|
||||
assert msg["type"] == "collective_rpc_result"
|
||||
assert msg["rpc_id"] == "rpc-1"
|
||||
assert msg["stage_ids"] == [1]
|
||||
assert msg["results"] == [{"stage": 1}]
|
||||
assert not stage0.collective_rpc_calls
|
||||
assert len(stage1.collective_rpc_calls) == 1
|
||||
assert "collective_rpc: ignoring invalid stage_id 99" in caplog.text
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_replica_cfg_companion_inherits_parent_affinity(orchestrator_factory) -> None:
|
||||
"""CFG companions should be routed to the same stage-0 replica as their parent."""
|
||||
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0_r0, stage0_r1]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[default_vllm_cfg],
|
||||
)
|
||||
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
|
||||
|
||||
try:
|
||||
# Consume replica-0 first so the parent request binds to replica-1.
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="warmup",
|
||||
prompt=SimpleNamespace(request_id="warmup", prompt_token_ids=[0]),
|
||||
original_prompt={"prompt": "warmup"},
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
|
||||
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="parent",
|
||||
prompt=SimpleNamespace(request_id="parent", prompt_token_ids=[1, 2]),
|
||||
original_prompt={"prompt": "parent"},
|
||||
sampling_params_list=[_sampling_params()],
|
||||
final_stage_id=0,
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 1)
|
||||
|
||||
orchestrator_fixture.request_sync_q.put_nowait(
|
||||
{
|
||||
"type": "add_companion_request",
|
||||
"companion_id": "parent-neg",
|
||||
"parent_id": "parent",
|
||||
"role": "negative",
|
||||
"prompt": SimpleNamespace(request_id="parent-neg", prompt_token_ids=[9]),
|
||||
"companion_prompt_text": {"prompt": "negative"},
|
||||
"sampling_params_list": [_sampling_params()],
|
||||
}
|
||||
)
|
||||
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 2)
|
||||
|
||||
assert stage_pools[0].get_bound_replica_id("parent") == 1
|
||||
assert stage_pools[0].get_bound_replica_id("parent-neg") == 1
|
||||
assert len(stage0_r0.add_request_calls) == 1
|
||||
assert stage0_r1.add_request_calls[0][0].request_id == "parent"
|
||||
assert stage0_r1.add_request_calls[1][0].request_id == "parent-neg"
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
@@ -124,8 +124,8 @@ async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_fa
|
||||
@pytest.mark.asyncio
|
||||
async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -> None:
|
||||
"""When a diffusion stage returns an OmniRequestOutput with a non-None
|
||||
error, the orchestrator must route it as a finished output message and
|
||||
clean up the request state.
|
||||
error, the orchestrator must route it as an error message and clean up
|
||||
the request state.
|
||||
"""
|
||||
stage0 = FakeStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
|
||||
orchestrator_fixture = orchestrator_factory([stage0])
|
||||
@@ -148,10 +148,10 @@ async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -
|
||||
|
||||
msg = await _get_any_output_message(orchestrator_fixture)
|
||||
|
||||
assert msg["type"] == "output"
|
||||
assert msg["type"] == "error"
|
||||
assert msg["request_id"] == "req-err"
|
||||
assert msg["finished"] is True
|
||||
assert msg["engine_outputs"].error == "gpu fault"
|
||||
assert msg["stage_id"] == 0
|
||||
assert msg["error"] == "gpu fault"
|
||||
|
||||
# Request state should be cleaned up.
|
||||
await _wait_for(lambda: "req-err" not in orchestrator_fixture.orchestrator.request_states)
|
||||
|
||||
@@ -7,18 +7,18 @@ from vllm import SamplingParams
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _DummySenderStage:
|
||||
stage_type = "llm"
|
||||
final_output = False
|
||||
|
||||
def __init__(self, sender_info):
|
||||
self._sender_info = sender_info
|
||||
self.engine_outputs = None
|
||||
|
||||
def set_engine_outputs(self, outputs):
|
||||
self.engine_outputs = outputs
|
||||
|
||||
def get_kv_sender_info(self):
|
||||
return self._sender_info
|
||||
@@ -26,6 +26,7 @@ class _DummySenderStage:
|
||||
|
||||
class _DummyDiffusionStage:
|
||||
stage_type = "diffusion"
|
||||
final_output = True
|
||||
custom_process_input_func = None
|
||||
|
||||
def __init__(self, engine_input_source=None):
|
||||
@@ -43,6 +44,15 @@ class _DummyDiffusionStage:
|
||||
)
|
||||
|
||||
|
||||
def _build_sender_pool(stage_id: int, sender_info: dict[str, object]) -> StagePool:
|
||||
return StagePool(
|
||||
stage_id,
|
||||
_DummySenderStage(sender_info),
|
||||
output_processor=object(),
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
)
|
||||
|
||||
|
||||
def test_stage_engine_core_client_builds_kv_sender_info_from_tcp_address():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
@@ -126,14 +136,13 @@ def test_stage_engine_core_client_preserves_explicit_loopback_sender_host():
|
||||
|
||||
def test_forward_to_diffusion_attaches_kv_sender_info():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
sender_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
|
||||
diffusion_pool = StagePool(1, diffusion_stage)
|
||||
|
||||
orchestrator.num_stages = 2
|
||||
orchestrator.stage_clients = [sender_stage, diffusion_stage]
|
||||
orchestrator.stage_pools = [sender_pool, diffusion_pool]
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
orchestrator.stage_vllm_configs = [None, None]
|
||||
orchestrator.output_processors = [None, None]
|
||||
|
||||
params = OmniDiffusionSamplingParams()
|
||||
req_state = OrchestratorRequestState(
|
||||
@@ -144,9 +153,8 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
|
||||
)
|
||||
|
||||
output = SimpleNamespace(request_id="req-1", finished=True)
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-1", 0, output, req_state))
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-1", sender_pool.stage_id, output, req_state))
|
||||
|
||||
assert sender_stage.engine_outputs == [output]
|
||||
assert diffusion_stage.calls[0]["request_id"] == "req-1"
|
||||
assert diffusion_stage.calls[0]["kv_sender_info"] == {
|
||||
0: {"host": "10.0.0.2", "zmq_port": 50151},
|
||||
@@ -156,15 +164,14 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
|
||||
|
||||
def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
source_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
|
||||
previous_stage = _DummySenderStage({"host": "10.0.0.9", "zmq_port": 59999})
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
source_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
|
||||
previous_pool = _build_sender_pool(1, {"host": "10.0.0.9", "zmq_port": 59999})
|
||||
diffusion_pool = StagePool(2, diffusion_stage)
|
||||
|
||||
orchestrator.num_stages = 3
|
||||
orchestrator.stage_clients = [source_stage, previous_stage, diffusion_stage]
|
||||
orchestrator.stage_pools = [source_pool, previous_pool, diffusion_pool]
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
orchestrator.stage_vllm_configs = [None, None, None]
|
||||
orchestrator.output_processors = [None, None, None]
|
||||
|
||||
params = OmniDiffusionSamplingParams()
|
||||
req_state = OrchestratorRequestState(
|
||||
@@ -175,9 +182,8 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
|
||||
)
|
||||
|
||||
output = SimpleNamespace(request_id="req-3", finished=True)
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-3", 1, output, req_state))
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-3", previous_pool.stage_id, output, req_state))
|
||||
|
||||
assert previous_stage.engine_outputs == [output]
|
||||
assert diffusion_stage.calls[0]["kv_sender_info"] == {
|
||||
0: {"host": "10.0.0.2", "zmq_port": 50151},
|
||||
}
|
||||
@@ -185,9 +191,10 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
|
||||
|
||||
def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
sender_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []
|
||||
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
|
||||
diffusion_pool = StagePool(1, diffusion_stage)
|
||||
|
||||
class _AsyncQueue:
|
||||
def __init__(self):
|
||||
@@ -197,10 +204,8 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
|
||||
self.items.append(item)
|
||||
|
||||
orchestrator.num_stages = 2
|
||||
orchestrator.stage_clients = [sender_stage, diffusion_stage]
|
||||
orchestrator.stage_pools = [sender_pool, diffusion_pool]
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
orchestrator.stage_vllm_configs = [None, None]
|
||||
orchestrator.output_processors = [None, None]
|
||||
orchestrator.output_async_queue = _AsyncQueue()
|
||||
orchestrator.request_states = {}
|
||||
orchestrator._pd_kv_params = {}
|
||||
@@ -217,7 +222,6 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
|
||||
output = SimpleNamespace(request_id="req-empty", finished=True)
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-empty", 0, output, req_state))
|
||||
|
||||
assert sender_stage.engine_outputs == [output]
|
||||
assert diffusion_stage.calls == []
|
||||
assert len(orchestrator.output_async_queue.items) == 1
|
||||
terminal_msg = orchestrator.output_async_queue.items[0]
|
||||
@@ -231,10 +235,11 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
|
||||
|
||||
def test_prewarm_diffusion_attaches_kv_sender_info():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
sender_stage = _DummySenderStage({"host": "10.0.0.3", "zmq_port": 50151})
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
sender_pool = _build_sender_pool(0, {"host": "10.0.0.3", "zmq_port": 50151})
|
||||
diffusion_pool = StagePool(1, diffusion_stage)
|
||||
|
||||
orchestrator.stage_clients = [sender_stage, diffusion_stage]
|
||||
orchestrator.stage_pools = [sender_pool, diffusion_pool]
|
||||
orchestrator.num_stages = 2
|
||||
|
||||
req_state = OrchestratorRequestState(
|
||||
|
||||
+663
-1328
File diff suppressed because it is too large.
Load diff
@@ -38,6 +38,7 @@ def _make_headless_args() -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
model="fake-model",
|
||||
stage_id=3,
|
||||
replica_id=0,
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
api_server_count=0,
|
||||
@@ -45,6 +46,7 @@ def _make_headless_args() -> argparse.Namespace:
|
||||
stage_configs_path=None,
|
||||
log_stats=False,
|
||||
disable_log_stats=False,
|
||||
stage_init_timeout=600,
|
||||
)
|
||||
|
||||
|
||||
@@ -59,6 +61,7 @@ def test_run_headless_registers_stage_once_and_launches_all_local_engines(mocker
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = mocker.Mock(parallel_config=parallel_config)
|
||||
vllm_config.needs_dp_coordinator = False
|
||||
executor_class = mocker.Mock()
|
||||
engine_manager = mocker.Mock()
|
||||
|
||||
@@ -99,6 +102,7 @@ def test_run_headless_registers_stage_once_and_launches_all_local_engines(mocker
|
||||
omni_stage_id=3,
|
||||
omni_stage_config=stage_cfg,
|
||||
coordinator=None,
|
||||
replica_id=0,
|
||||
)
|
||||
mock_manager_cls.assert_called_once()
|
||||
manager_kwargs = mock_manager_cls.call_args.kwargs
|
||||
@@ -155,13 +159,56 @@ def test_run_headless_honors_explicit_log_stats_flag(mocker: MockerFixture) -> N
|
||||
assert manager_kwargs["log_stats"] is True
|
||||
|
||||
|
||||
def test_run_headless_registers_llm_replica_id(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
args.replica_id = 2
|
||||
stage_cfg = mocker.Mock(stage_id=3)
|
||||
parallel_config = mocker.Mock(
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
data_parallel_rank_local=0,
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = mocker.Mock(parallel_config=parallel_config)
|
||||
vllm_config.needs_dp_coordinator = False
|
||||
engine_manager = mocker.Mock()
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", [stage_cfg]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
return_value=(vllm_config, mocker.Mock()),
|
||||
)
|
||||
mock_register = mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value="tcp://127.0.0.1:26001",
|
||||
)
|
||||
mocker.patch("vllm.v1.engine.utils.CoreEngineProcManager", return_value=engine_manager)
|
||||
mocker.patch("signal.signal")
|
||||
|
||||
run_headless(args)
|
||||
|
||||
assert mock_register.call_args.kwargs["replica_id"] == 2
|
||||
|
||||
|
||||
def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
args.replica_id = 1
|
||||
stage_cfg = mocker.Mock(stage_id=3, stage_type="diffusion")
|
||||
stage_cfg.engine_args = mocker.Mock()
|
||||
stage_cfg.engine_input_source = []
|
||||
stage_cfgs = [stage_cfg]
|
||||
metadata = mocker.Mock(stage_id=3)
|
||||
metadata = mocker.Mock(stage_id=3, stage_type="diffusion")
|
||||
od_config = mocker.Mock()
|
||||
proc = mocker.Mock()
|
||||
proc.exitcode = 0
|
||||
@@ -199,6 +246,7 @@ def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFix
|
||||
omni_stage_id=3,
|
||||
omni_stage_config=stage_cfg,
|
||||
return_addresses=True,
|
||||
replica_id=1,
|
||||
)
|
||||
mock_spawn.assert_called_once_with(
|
||||
"fake-model",
|
||||
@@ -207,5 +255,5 @@ def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFix
|
||||
request_address="tcp://127.0.0.1:26002",
|
||||
response_address="tcp://127.0.0.1:26003",
|
||||
)
|
||||
mock_handshake.assert_called_once_with(proc, "tcp://127.0.0.1:26001")
|
||||
mock_handshake.assert_called_once_with(proc, "tcp://127.0.0.1:26001", 600)
|
||||
proc.join.assert_called_once_with()
|
||||
@@ -360,6 +360,47 @@ class TestLoadAndResolveStageConfigs:
|
||||
assert len(stage_configs) == 1
|
||||
assert "dtype" in stage_configs[0]["engine_args"]
|
||||
|
||||
def test_stage_configs_path_promotes_new_deploy_yaml_without_expanding_replicas(
|
||||
self, tmp_path, mocker: MockerFixture
|
||||
):
|
||||
deploy_path = tmp_path / "qwen3_multi.yaml"
|
||||
deploy_path.write_text(
|
||||
'stages:\n - stage_id: 0\n devices: "0"\n - stage_id: 1\n devices: "1,2,3"\n num_replicas: 3\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
returned_stage_configs = [
|
||||
create_config({"stage_id": 0, "runtime": {"devices": "0"}, "engine_args": {"model": "dummy"}}),
|
||||
create_config(
|
||||
{
|
||||
"stage_id": 1,
|
||||
"runtime": {"devices": "1,2,3", "num_replicas": 3},
|
||||
"engine_args": {"model": "dummy"},
|
||||
}
|
||||
),
|
||||
]
|
||||
load_stage_configs = mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
|
||||
return_value=returned_stage_configs,
|
||||
)
|
||||
|
||||
config_path, stage_configs = load_and_resolve_stage_configs(
|
||||
model="dummy-model",
|
||||
stage_configs_path=str(deploy_path),
|
||||
kwargs={},
|
||||
)
|
||||
|
||||
load_stage_configs.assert_called_once_with(
|
||||
"dummy-model",
|
||||
base_engine_args={},
|
||||
deploy_config_path=str(deploy_path),
|
||||
stage_overrides=None,
|
||||
)
|
||||
assert config_path == str(deploy_path)
|
||||
assert len(stage_configs) == 2
|
||||
assert stage_configs[1].runtime.num_replicas == 3
|
||||
assert stage_configs[1].runtime.devices == "1,2,3"
|
||||
|
||||
|
||||
class TestLoadStageConfigsFromYaml:
|
||||
"""Regression tests for stage-config loading and merging."""
|
||||
|
||||
+64
-28
@@ -385,7 +385,8 @@ class OmniServerStageCli(OmniServer):
|
||||
self.stage_ids = stage_ids or self._load_stage_ids(resolved_cfg)
|
||||
if 0 not in self.stage_ids:
|
||||
raise ValueError(f"Stage CLI test requires stage_id=0 in config: {stage_config_path}")
|
||||
self.stage_procs: dict[int, subprocess.Popen] = {}
|
||||
self.stage_replica_counts = self._load_stage_replica_counts(resolved_cfg)
|
||||
self.stage_procs: dict[tuple[int, int], subprocess.Popen] = {}
|
||||
self.proc = None
|
||||
|
||||
@staticmethod
|
||||
@@ -415,6 +416,19 @@ class OmniServerStageCli(OmniServer):
|
||||
runtime_devices[int(stage_id)] = str(devices)
|
||||
return runtime_devices
|
||||
|
||||
@staticmethod
|
||||
def _load_stage_replica_counts(resolved_config: dict) -> dict[int, int]:
|
||||
replica_counts: dict[int, int] = {}
|
||||
for stage in OmniServerStageCli._stage_entries(resolved_config):
|
||||
stage_id = stage.get("stage_id")
|
||||
if stage_id is None:
|
||||
continue
|
||||
replica_counts[int(stage_id)] = max(
|
||||
1,
|
||||
int(stage.get("num_replicas") or stage.get("runtime", {}).get("num_replicas", 1)),
|
||||
)
|
||||
return replica_counts
|
||||
|
||||
@classmethod
|
||||
def _parse_device_list(cls, devices: str | int) -> list[str]:
|
||||
if isinstance(devices, int):
|
||||
@@ -452,13 +466,29 @@ class OmniServerStageCli(OmniServer):
|
||||
|
||||
return ",".join(visible_device_list[idx] for idx in logical_ids)
|
||||
|
||||
def _set_stage_device_env(self, stage_id: int, env: dict[str, str], devices: str) -> None:
|
||||
mapped_devices = self._map_stage_devices(stage_id, self.visible_device_list, devices)
|
||||
def _devices_for_replica(self, stage_id: int, devices: str, replica_id: int) -> str:
|
||||
replica_count = self.stage_replica_counts.get(stage_id, 1)
|
||||
if replica_count == 1:
|
||||
return devices
|
||||
|
||||
device_list = self._parse_device_list(devices)
|
||||
if len(device_list) % replica_count != 0:
|
||||
raise ValueError(
|
||||
f"Stage {stage_id} has {len(device_list)} device(s) for {replica_count} replica(s); "
|
||||
"device count must be divisible by replica count"
|
||||
)
|
||||
devices_per_replica = len(device_list) // replica_count
|
||||
start = replica_id * devices_per_replica
|
||||
return ",".join(device_list[start : start + devices_per_replica])
|
||||
|
||||
def _set_stage_device_env(self, stage_id: int, env: dict[str, str], devices: str, replica_id: int = 0) -> None:
|
||||
replica_devices = self._devices_for_replica(stage_id, devices, replica_id)
|
||||
mapped_devices = self._map_stage_devices(stage_id, self.visible_device_list, replica_devices)
|
||||
env_var = getattr(current_omni_platform, "device_control_env_var", None)
|
||||
if env_var:
|
||||
env[env_var] = mapped_devices
|
||||
|
||||
def _build_stage_cmd(self, stage_id: int, *, headless: bool) -> list[str]:
|
||||
def _build_stage_cmd(self, stage_id: int, *, headless: bool, replica_id: int = 0) -> list[str]:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
@@ -474,6 +504,8 @@ class OmniServerStageCli(OmniServer):
|
||||
self.host,
|
||||
"--omni-master-port",
|
||||
str(self.master_port),
|
||||
"--replica-id",
|
||||
str(replica_id),
|
||||
]
|
||||
|
||||
if headless:
|
||||
@@ -484,7 +516,7 @@ class OmniServerStageCli(OmniServer):
|
||||
cmd += self.serve_args
|
||||
return cmd
|
||||
|
||||
def _launch_stage(self, stage_id: int, *, headless: bool) -> None:
|
||||
def _launch_stage(self, stage_id: int, *, headless: bool, replica_id: int = 0) -> None:
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if self.env_dict is not None:
|
||||
@@ -492,19 +524,20 @@ class OmniServerStageCli(OmniServer):
|
||||
|
||||
devices = self.stage_runtime_devices.get(stage_id)
|
||||
if devices:
|
||||
self._set_stage_device_env(stage_id, env, devices)
|
||||
self._set_stage_device_env(stage_id, env, devices, replica_id=replica_id)
|
||||
|
||||
cmd = self._build_stage_cmd(stage_id, headless=headless)
|
||||
print(f"Launching OmniServerStageCli stage {stage_id}: {' '.join(cmd)}")
|
||||
cmd = self._build_stage_cmd(stage_id, headless=headless, replica_id=replica_id)
|
||||
print(f"Launching OmniServerStageCli stage {stage_id} replica {replica_id}: {' '.join(cmd)}")
|
||||
# Capture each subprocess's stdout+stderr to a per-stage log file so
|
||||
# debugging "Stage N exited before API server ready" doesn't rely on
|
||||
# guessing; the file is surfaced in the RuntimeError message.
|
||||
log_path = Path(tempfile.gettempdir()) / f"omni_stage_{stage_id}_{self.master_port}.log"
|
||||
log_path = Path(tempfile.gettempdir()) / f"omni_stage_{stage_id}_replica_{replica_id}_{self.master_port}.log"
|
||||
self._stage_log_paths = getattr(self, "_stage_log_paths", {})
|
||||
self._stage_log_paths[stage_id] = log_path
|
||||
stage_key = (stage_id, replica_id)
|
||||
self._stage_log_paths[stage_key] = log_path
|
||||
log_fh = open(log_path, "w", buffering=1) # noqa: SIM115 - closed in __exit__
|
||||
self._stage_log_files = getattr(self, "_stage_log_files", {})
|
||||
self._stage_log_files[stage_id] = log_fh
|
||||
self._stage_log_files[stage_key] = log_fh
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=env,
|
||||
@@ -512,36 +545,39 @@ class OmniServerStageCli(OmniServer):
|
||||
stdout=log_fh,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
self.stage_procs[stage_id] = proc
|
||||
if stage_id == 0:
|
||||
self.stage_procs[stage_key] = proc
|
||||
if stage_id == 0 and replica_id == 0:
|
||||
self.proc = proc
|
||||
|
||||
def _ensure_stage_processes_alive(self) -> None:
|
||||
for stage_id, proc in self.stage_procs.items():
|
||||
for (stage_id, replica_id), proc in self.stage_procs.items():
|
||||
ret = proc.poll()
|
||||
if ret is not None:
|
||||
log_path = getattr(self, "_stage_log_paths", {}).get(stage_id)
|
||||
log_path = getattr(self, "_stage_log_paths", {}).get((stage_id, replica_id))
|
||||
tail = ""
|
||||
if log_path and log_path.exists():
|
||||
try:
|
||||
with open(log_path, encoding="utf-8", errors="replace") as f:
|
||||
lines = f.readlines()
|
||||
tail = "\n=== Last 60 lines of stage {} log ({}) ===\n{}".format(
|
||||
stage_id, log_path, "".join(lines[-60:]) or "<empty>"
|
||||
tail = "\n=== Last 60 lines of stage {} replica {} log ({}) ===\n{}".format(
|
||||
stage_id, replica_id, log_path, "".join(lines[-60:]) or "<empty>"
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - diagnostic only
|
||||
tail = f"\n<failed to read stage log {log_path}: {exc}>"
|
||||
raise RuntimeError(f"Stage {stage_id} exited with code {ret} before API server became ready.{tail}")
|
||||
raise RuntimeError(
|
||||
f"Stage {stage_id} replica {replica_id} exited with code {ret} before API server became ready.{tail}"
|
||||
)
|
||||
|
||||
def _start_server(self) -> None:
|
||||
ordered_stage_ids = [0, *[stage_id for stage_id in self.stage_ids if stage_id != 0]]
|
||||
|
||||
self._launch_stage(0, headless=False)
|
||||
self._launch_stage(0, headless=False, replica_id=0)
|
||||
time.sleep(2)
|
||||
self._ensure_stage_processes_alive()
|
||||
|
||||
for stage_id in ordered_stage_ids[1:]:
|
||||
self._launch_stage(stage_id, headless=True)
|
||||
for replica_id in range(self.stage_replica_counts.get(stage_id, 1)):
|
||||
self._launch_stage(stage_id, headless=True, replica_id=replica_id)
|
||||
|
||||
max_wait = 1200
|
||||
start_time = time.time()
|
||||
@@ -568,15 +604,15 @@ class OmniServerStageCli(OmniServer):
|
||||
whatever state the stage was in when it was torn down.
|
||||
"""
|
||||
log_paths = getattr(self, "_stage_log_paths", {}) or {}
|
||||
for stage_id in sorted(log_paths):
|
||||
log_path = log_paths[stage_id]
|
||||
for stage_id, replica_id in sorted(log_paths):
|
||||
log_path = log_paths[(stage_id, replica_id)]
|
||||
if not log_path or not log_path.exists():
|
||||
continue
|
||||
try:
|
||||
with open(log_path, encoding="utf-8", errors="replace") as f:
|
||||
lines = f.readlines()
|
||||
except Exception as exc: # pragma: no cover - diagnostic only
|
||||
print(f"[OmniServerStageCli] stage {stage_id} log read failed: {exc}", flush=True)
|
||||
print(f"[OmniServerStageCli] stage {stage_id} replica {replica_id} log read failed: {exc}", flush=True)
|
||||
continue
|
||||
total = len(lines)
|
||||
if total <= head_lines + tail_lines:
|
||||
@@ -587,18 +623,18 @@ class OmniServerStageCli(OmniServer):
|
||||
head_chunk = lines[:head_lines]
|
||||
tail_chunk = lines[-tail_lines:]
|
||||
elided = total - head_lines - tail_lines
|
||||
print(f"\n=== stage {stage_id} log HEAD ({log_path}) ===", flush=True)
|
||||
print(f"\n=== stage {stage_id} replica {replica_id} log HEAD ({log_path}) ===", flush=True)
|
||||
print("".join(head_chunk).rstrip("\n"), flush=True)
|
||||
if tail_chunk:
|
||||
print(f"\n... [{elided} lines elided] ...", flush=True)
|
||||
print(f"\n=== stage {stage_id} log TAIL ({log_path}) ===", flush=True)
|
||||
print(f"\n=== stage {stage_id} replica {replica_id} log TAIL ({log_path}) ===", flush=True)
|
||||
print("".join(tail_chunk).rstrip("\n"), flush=True)
|
||||
print(f"=== end stage {stage_id} log ===\n", flush=True)
|
||||
print(f"=== end stage {stage_id} replica {replica_id} log ===\n", flush=True)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self._dump_stage_logs_for_debug()
|
||||
for stage_id in sorted(self.stage_procs, reverse=True):
|
||||
proc = self.stage_procs[stage_id]
|
||||
for stage_key in sorted(self.stage_procs, reverse=True):
|
||||
proc = self.stage_procs[stage_key]
|
||||
if proc.poll() is None:
|
||||
self._kill_process_tree(proc.pid)
|
||||
run_forced_gpu_cleanup_round()
|
||||
|
||||
@@ -387,6 +387,79 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
|
||||
},
|
||||
},
|
||||
},
|
||||
"qwen3_omni_moe_multi_replicas_4gpu": {
|
||||
"base_config": "qwen3_omni_moe.yaml",
|
||||
"async_chunk": True,
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"devices": "0",
|
||||
"gpu_memory_utilization": 0.85,
|
||||
"max_num_seqs": 6,
|
||||
"max_model_len": 32768,
|
||||
"mm_processor_cache_gb": 0,
|
||||
"load_format": "dummy",
|
||||
"default_sampling_params": {"max_tokens": 150, "ignore_eos": False},
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"devices": "1,2,3",
|
||||
"num_replicas": 3,
|
||||
"gpu_memory_utilization": 0.6,
|
||||
"max_num_seqs": 2,
|
||||
"max_model_len": 32768,
|
||||
"load_format": "dummy",
|
||||
"default_sampling_params": {"max_tokens": 1000},
|
||||
},
|
||||
{
|
||||
"stage_id": 2,
|
||||
"devices": "1,2,3",
|
||||
"num_replicas": 3,
|
||||
"gpu_memory_utilization": 0.1,
|
||||
"max_num_seqs": 2,
|
||||
"max_num_batched_tokens": 65536,
|
||||
"load_format": "dummy",
|
||||
"default_sampling_params": {"max_tokens": 2000},
|
||||
},
|
||||
],
|
||||
},
|
||||
"bagel_multi_replicas_4gpu": {
|
||||
"base_config": "bagel.yaml",
|
||||
"async_chunk": False,
|
||||
"stages": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"devices": "0",
|
||||
"max_num_seqs": 6,
|
||||
"max_num_batched_tokens": 16384,
|
||||
"gpu_memory_utilization": 0.45,
|
||||
"load_format": "dummy",
|
||||
"default_sampling_params": {
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"max_tokens": 256,
|
||||
"detokenize": False,
|
||||
},
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"devices": "1,2,3",
|
||||
"num_replicas": 3,
|
||||
"max_num_seqs": 1,
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.7,
|
||||
"load_format": "dummy",
|
||||
"default_sampling_params": {
|
||||
"seed": 42,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
"bagel": {
|
||||
"base_config": "bagel.yaml",
|
||||
"stages": [
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Tests for OmniServerStageCli process planning helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from tests.helpers.runtime import OmniServerStageCli
|
||||
|
||||
|
||||
def _write_stage_config(tmp_path: Path) -> str:
|
||||
path = tmp_path / "stages.yaml"
|
||||
path.write_text(
|
||||
"""
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
- stage_id: 1
|
||||
devices: "1,2,3"
|
||||
num_replicas: 3
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return str(path)
|
||||
|
||||
|
||||
def test_stage_cli_builds_headless_replica_cmd(tmp_path: Path) -> None:
|
||||
server = OmniServerStageCli("fake-model", _write_stage_config(tmp_path), ["--disable-log-stats"])
|
||||
|
||||
cmd = server._build_stage_cmd(1, headless=True, replica_id=2)
|
||||
|
||||
assert "--headless" in cmd
|
||||
assert cmd[cmd.index("--stage-id") + 1] == "1"
|
||||
assert cmd[cmd.index("--replica-id") + 1] == "2"
|
||||
assert cmd[cmd.index("--stage-configs-path") + 1] == server.stage_config_path
|
||||
|
||||
|
||||
def test_stage_cli_splits_devices_per_replica(tmp_path: Path) -> None:
|
||||
server = OmniServerStageCli("fake-model", _write_stage_config(tmp_path), [])
|
||||
|
||||
assert server._devices_for_replica(1, "1,2,3", 0) == "1"
|
||||
assert server._devices_for_replica(1, "1,2,3", 1) == "2"
|
||||
assert server._devices_for_replica(1, "1,2,3", 2) == "3"
|
||||
|
||||
|
||||
def test_stage_cli_maps_visible_devices_after_replica_split(tmp_path: Path) -> None:
|
||||
server = OmniServerStageCli(
|
||||
"fake-model",
|
||||
_write_stage_config(tmp_path),
|
||||
[],
|
||||
env_dict={"CUDA_VISIBLE_DEVICES": "4,5,6,7"},
|
||||
)
|
||||
env: dict[str, str] = {}
|
||||
|
||||
server._set_stage_device_env(1, env, "1,2,3", replica_id=1)
|
||||
|
||||
assert env["CUDA_VISIBLE_DEVICES"] == "6"
|
||||
+4
-8
@@ -44,16 +44,12 @@ def _transfer_manager(
|
||||
|
||||
|
||||
def test_text2flow_supports_batched_source_outputs():
|
||||
stage_list = [
|
||||
SimpleNamespace(
|
||||
engine_outputs=[
|
||||
_source_output("req-0", [10, 11], [1, 2, 3], {"speech_token": torch.tensor([[1, 2]])}),
|
||||
_source_output("req-1", [20, 21], [4, 5], {"speech_token": torch.tensor([[3, 4]])}),
|
||||
]
|
||||
)
|
||||
source_outputs = [
|
||||
_source_output("req-0", [10, 11], [1, 2, 3], {"speech_token": torch.tensor([[1, 2]])}),
|
||||
_source_output("req-1", [20, 21], [4, 5], {"speech_token": torch.tensor([[3, 4]])}),
|
||||
]
|
||||
|
||||
outputs = text2flow(stage_list=stage_list, engine_input_source=[0], prompt=None)
|
||||
outputs = text2flow(source_outputs=source_outputs, prompt=None)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert outputs[0]["prompt_token_ids"] == [1, 2, 3]
|
||||
|
||||
@@ -32,11 +32,6 @@ def _source_output(token_ids: list[int], mm_output: dict | None = None):
|
||||
)
|
||||
|
||||
|
||||
def _stage_with_outputs(outputs):
|
||||
"""Create a stage list entry with engine_outputs."""
|
||||
return SimpleNamespace(engine_outputs=outputs)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tests for _has_source_image
|
||||
# =============================================================================
|
||||
@@ -285,11 +280,11 @@ class TestAr2Diffusion:
|
||||
"""Test basic text-to-image pipeline: AR -> Diffusion."""
|
||||
# 1024x1024 t2i: small(256) + large(1024) + EOS
|
||||
token_ids = list(range(256)) + list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
|
||||
source_outputs = [_source_output(token_ids)]
|
||||
|
||||
prompt = {"prompt": "a cat", "mm_processor_kwargs": {"target_h": 1024, "target_w": 1024}}
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt])
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt])
|
||||
assert len(result) == 1
|
||||
assert result[0]["prompt"] == "a cat"
|
||||
assert result[0]["height"] == 1024
|
||||
@@ -300,7 +295,7 @@ class TestAr2Diffusion:
|
||||
"""Test image-to-image with prior_token_image_ids from AR model."""
|
||||
token_ids = list(range(1024)) + [16385]
|
||||
mm_output = {"ids": {"prior_image": torch.tensor([1, 2, 3])}}
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids, mm_output)])]
|
||||
source_outputs = [_source_output(token_ids, mm_output)]
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -311,14 +306,14 @@ class TestAr2Diffusion:
|
||||
"multi_modal_data": {"image": img},
|
||||
}
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt])
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt])
|
||||
assert len(result) == 1
|
||||
assert result[0]["extra"]["prior_token_image_ids"] is not None
|
||||
|
||||
def test_i2i_detected_via_modalities(self):
|
||||
"""Test i2i mode detected via modalities field."""
|
||||
token_ids = list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
|
||||
source_outputs = [_source_output(token_ids)]
|
||||
|
||||
prompt = {
|
||||
"prompt": "edit this",
|
||||
@@ -326,35 +321,26 @@ class TestAr2Diffusion:
|
||||
"modalities": ["img2img"],
|
||||
}
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt])
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt])
|
||||
assert len(result) == 1
|
||||
|
||||
def test_empty_engine_input_source_raises(self):
|
||||
with pytest.raises(ValueError, match="engine_input_source cannot be empty"):
|
||||
ar2diffusion([], [], prompt={})
|
||||
|
||||
def test_invalid_stage_id_raises(self):
|
||||
with pytest.raises(IndexError, match="Invalid stage_id"):
|
||||
ar2diffusion([_stage_with_outputs(None)], [5], prompt={})
|
||||
|
||||
def test_no_outputs_raises(self):
|
||||
with pytest.raises(RuntimeError, match="has no outputs yet"):
|
||||
ar2diffusion([SimpleNamespace(engine_outputs=None)], [0], prompt={})
|
||||
def test_empty_source_outputs_returns_empty_list(self):
|
||||
assert ar2diffusion([], prompt={}) == []
|
||||
|
||||
def test_default_dimensions(self):
|
||||
"""When no height/width in prompt, defaults to 1024x1024."""
|
||||
token_ids = list(range(256)) + list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
|
||||
source_outputs = [_source_output(token_ids)]
|
||||
|
||||
prompt = {"prompt": "test"}
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt])
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt])
|
||||
assert result[0]["height"] == 1024
|
||||
assert result[0]["width"] == 1024
|
||||
|
||||
def test_requires_multimodal_data_with_pil_image(self):
|
||||
"""Test that pil_image is included when requires_multimodal_data=True."""
|
||||
token_ids = list(range(256)) + list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
|
||||
source_outputs = [_source_output(token_ids)]
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -364,13 +350,13 @@ class TestAr2Diffusion:
|
||||
"multi_modal_data": {"image": img},
|
||||
}
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt], requires_multimodal_data=True)
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt], requires_multimodal_data=True)
|
||||
assert result[0]["pil_image"] is img
|
||||
|
||||
def test_extra_params_passed_through(self):
|
||||
"""Test that seed, num_inference_steps, guidance_scale, negative_prompt are passed."""
|
||||
token_ids = list(range(256)) + list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
|
||||
source_outputs = [_source_output(token_ids)]
|
||||
|
||||
prompt = {
|
||||
"prompt": "test",
|
||||
@@ -380,7 +366,7 @@ class TestAr2Diffusion:
|
||||
"negative_prompt": "blurry",
|
||||
}
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=[prompt])
|
||||
result = ar2diffusion(source_outputs, prompt=[prompt])
|
||||
assert result[0]["seed"] == 42
|
||||
assert result[0]["num_inference_steps"] == 50
|
||||
assert result[0]["guidance_scale"] == 7.5
|
||||
@@ -390,14 +376,14 @@ class TestAr2Diffusion:
|
||||
"""Test processing multiple requests in a batch."""
|
||||
tokens1 = list(range(256)) + list(range(1024)) + [16385]
|
||||
tokens2 = list(range(256)) + list(range(1024)) + [16385]
|
||||
stage_list = [_stage_with_outputs([_source_output(tokens1), _source_output(tokens2)])]
|
||||
source_outputs = [_source_output(tokens1), _source_output(tokens2)]
|
||||
|
||||
prompts = [
|
||||
{"prompt": "first", "mm_processor_kwargs": {"target_h": 1024, "target_w": 1024}},
|
||||
{"prompt": "second", "mm_processor_kwargs": {"target_h": 512, "target_w": 512}},
|
||||
]
|
||||
|
||||
result = ar2diffusion(stage_list, [0], prompt=prompts)
|
||||
result = ar2diffusion(source_outputs, prompt=prompts)
|
||||
assert len(result) == 2
|
||||
assert result[0]["prompt"] == "first"
|
||||
assert result[1]["prompt"] == "second"
|
||||
@@ -17,12 +17,11 @@ from vllm_omni.model_executor.stage_input_processors.mimo_audio import (
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_stage_list(codec_codes: torch.Tensor, request_id: str = "req-0"):
|
||||
"""Build a minimal stage_list[0] with a single talker output carrying codec_codes."""
|
||||
def _make_source_outputs(codec_codes: torch.Tensor, request_id: str = "req-0"):
|
||||
output = SimpleNamespace(multimodal_output={"codes": {"audio": codec_codes}})
|
||||
"""Build minimal source_outputs with one talker output carrying codec codes."""
|
||||
talker_output = SimpleNamespace(outputs=[output], request_id=request_id)
|
||||
stage0 = SimpleNamespace(engine_outputs=[talker_output])
|
||||
return [stage0]
|
||||
return [talker_output]
|
||||
|
||||
|
||||
def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
|
||||
@@ -33,7 +32,7 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
|
||||
frames = (MAX_CODE2WAV_TOKENS // 36) + 100
|
||||
codec_codes = torch.ones(frames, 1, 8, 4, dtype=torch.long)
|
||||
|
||||
stage_list = _make_stage_list(codec_codes, request_id="req-long")
|
||||
source_outputs = _make_source_outputs(codec_codes, request_id="req-long")
|
||||
|
||||
# Attach caplog's handler directly to the module logger so the warning is
|
||||
# captured regardless of propagation (vllm's logger configuration can
|
||||
@@ -43,7 +42,7 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
|
||||
prev_level = target_logger.level
|
||||
target_logger.setLevel(logging.WARNING)
|
||||
try:
|
||||
prompts = llm2code2wav(stage_list, engine_input_source=[0])
|
||||
prompts = llm2code2wav(source_outputs)
|
||||
finally:
|
||||
target_logger.removeHandler(caplog.handler)
|
||||
target_logger.setLevel(prev_level)
|
||||
@@ -58,9 +57,9 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
|
||||
def test_llm2code2wav_short_sequence_unchanged():
|
||||
"""Short codec sequences are returned without truncation."""
|
||||
codec_codes = torch.ones(4, 1, 8, 4, dtype=torch.long)
|
||||
stage_list = _make_stage_list(codec_codes, request_id="req-short")
|
||||
source_outputs = _make_source_outputs(codec_codes, request_id="req-short")
|
||||
|
||||
prompts = llm2code2wav(stage_list, engine_input_source=[0])
|
||||
prompts = llm2code2wav(source_outputs)
|
||||
|
||||
assert len(prompts) == 1
|
||||
# 4 frames + 1 pad row, flattened col-major → well below the cap
|
||||
|
||||
@@ -342,7 +342,7 @@ def test_non_async_processor_prepends_ref_code_and_sets_trim_context():
|
||||
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
|
||||
)
|
||||
|
||||
prompts = talker2code2wav(stage_list=[stage], engine_input_source=[0])
|
||||
prompts = talker2code2wav(stage.engine_outputs)
|
||||
|
||||
assert len(prompts) == 1
|
||||
prompt = prompts[0]
|
||||
@@ -388,7 +388,7 @@ def test_non_async_processor_filters_out_of_range_codec_values():
|
||||
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
|
||||
)
|
||||
|
||||
prompts = talker2code2wav(stage_list=[stage], engine_input_source=[0])
|
||||
prompts = talker2code2wav(stage.engine_outputs)
|
||||
|
||||
assert len(prompts) == 1
|
||||
prompt = prompts[0]
|
||||
|
||||
@@ -38,7 +38,7 @@ class TestGenerator2Tokenizer:
|
||||
audio = torch.tensor([10, 20, 30, 40])
|
||||
stage = _make_stage([_make_output([audio])])
|
||||
|
||||
result = generator2tokenizer([stage], engine_input_source=[0])
|
||||
result = generator2tokenizer(stage.engine_outputs)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["prompt_token_ids"] == [10, 20, 30, 40]
|
||||
@@ -49,27 +49,15 @@ class TestGenerator2Tokenizer:
|
||||
audio = torch.tensor([[1, 2, 3], [4, 5, 6]]) # (2, 3)
|
||||
stage = _make_stage([_make_output([audio])])
|
||||
|
||||
result = generator2tokenizer([stage], engine_input_source=[0])
|
||||
result = generator2tokenizer(stage.engine_outputs)
|
||||
|
||||
assert result[0]["prompt_token_ids"] == [1, 2, 3, 4, 5, 6]
|
||||
|
||||
def test_empty_engine_input_source_raises(self):
|
||||
"""Empty engine_input_source raises ValueError."""
|
||||
stage = _make_stage([_make_output([torch.tensor([1])])])
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
generator2tokenizer([stage], engine_input_source=[])
|
||||
|
||||
def test_invalid_stage_id_raises(self):
|
||||
"""Stage id beyond stage_list length raises IndexError."""
|
||||
stage = _make_stage([_make_output([torch.tensor([1])])])
|
||||
with pytest.raises(IndexError, match="Invalid stage_id"):
|
||||
generator2tokenizer([stage], engine_input_source=[5])
|
||||
|
||||
def test_no_outputs_yet_raises(self):
|
||||
"""Stage with engine_outputs=None raises RuntimeError."""
|
||||
stage = _make_stage(engine_outputs=None)
|
||||
with pytest.raises(RuntimeError, match="no outputs yet"):
|
||||
generator2tokenizer([stage], engine_input_source=[0])
|
||||
with pytest.raises(TypeError):
|
||||
generator2tokenizer(stage.engine_outputs)
|
||||
|
||||
|
||||
# ---- Helpers for generator2tokenizer_async_chunk (streaming) ----
|
||||
|
||||
@@ -920,6 +920,27 @@ class TestDeployConfigLoading:
|
||||
assert s0.yaml_engine_args["engine_output_type"] == "latent"
|
||||
assert s0.yaml_extras["default_sampling_params"]["detokenize"] is True
|
||||
|
||||
def test_merge_pipeline_deploy_preserves_num_replicas(self, tmp_path):
|
||||
from pathlib import Path
|
||||
|
||||
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
|
||||
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
|
||||
|
||||
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
|
||||
base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
|
||||
if not base.exists():
|
||||
pytest.skip("Deploy config not found")
|
||||
|
||||
overlay = tmp_path / "multi_replicas.yaml"
|
||||
overlay.write_text(f'base_config: {base}\nstages:\n - stage_id: 1\n devices: "1,2"\n num_replicas: 2\n')
|
||||
|
||||
deploy = load_deploy_config(overlay)
|
||||
assert deploy.stages[1].num_replicas == 2
|
||||
|
||||
stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
assert stages[1].yaml_runtime["devices"] == "1,2"
|
||||
assert stages[1].yaml_runtime["num_replicas"] == 2
|
||||
|
||||
|
||||
class TestQwen3OmniPipeline:
|
||||
def test_registered(self):
|
||||
|
||||
@@ -354,15 +354,15 @@ class TestFinishedLoadReqsDrain(unittest.TestCase):
|
||||
|
||||
|
||||
class TestLoadCustomFuncSelection(unittest.TestCase):
|
||||
def test_skips_legacy_stage_list_processors_for_full_payload_mode(self):
|
||||
legacy_paths = [
|
||||
def test_skips_non_payload_stage_input_processors_for_full_payload_mode(self):
|
||||
incompatible_paths = [
|
||||
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav",
|
||||
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2dit",
|
||||
"vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow",
|
||||
"vllm_omni.model_executor.stage_input_processors.glm_image.ar2diffusion",
|
||||
]
|
||||
|
||||
for func_path in legacy_paths:
|
||||
for func_path in incompatible_paths:
|
||||
selected_path, func = MixinHost._load_custom_func(
|
||||
SimpleNamespace(
|
||||
async_chunk=False,
|
||||
|
||||
Reference in new issue
Block a user