[FEAT] support multi-stage deployment (#2396)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
authored and GitHub committed 2026-05-06 19:33:18 +08:00
1 parent e969d2e99f
commit 1e5f288a91
55 files changed
+4684 -3407

No files matched your search

@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
@@ -94,3 +95,24 @@ def test_inline_shutdown(client, mock_engine):
assert client._shutting_down
mock_engine.close.assert_called_once()
def test_inline_client_requires_replica_id(mock_engine):
metadata = SimpleNamespace(
stage_id=0,
final_output=True,
final_output_type="image",
default_sampling_params={},
requires_multimodal_data=False,
custom_process_input_func=None,
engine_input_source=[],
)
with patch.object(InlineStageDiffusionClient, "_enrich_config"):
od_config = MagicMock(spec=OmniDiffusionConfig)
with pytest.raises(AttributeError, match="replica_id"):
InlineStageDiffusionClient(
model="test_model",
od_config=od_config,
metadata=metadata,
batch_size=1,
)
@@ -621,6 +621,29 @@ class TestStageDiffusionClientErrorPropagation:
assert client._shutting_down is True
assert client._engine_dead is True
def test_initialize_client_requires_replica_id(self):
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
client = object.__new__(StageDiffusionClient)
metadata = SimpleNamespace(
stage_id=0,
final_output=True,
final_output_type="image",
default_sampling_params=None,
requires_multimodal_data=False,
custom_process_input_func=None,
engine_input_source=[],
)
with pytest.raises(AttributeError, match="replica_id"):
client._initialize_client(
metadata,
"tcp://req",
"tcp://resp",
proc=None,
batch_size=1,
)
# ───────── monitor thread & death sentinel integration tests ─────────
@@ -0,0 +1,55 @@
"""Nightly-only e2e coverage for BAGEL diffusion multi-replica serving.
This test needs 4 H100 GPUs; keep it out of generic test_bagel_* jobs.
"""
import os
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/bagel_multi_replicas_4gpu.yaml")
ROUTE_STRESS_REQUESTS = 6
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
test_params = [
OmniServerParams(
model=MODEL,
stage_config_path=MULTI_REPLICA_DEPLOY,
server_args=["--disable-log-stats"],
use_stage_cli=True,
)
]
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_multi_stage_diffusion_uses_multi_replica_dit_stage(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": dummy_messages_from_mix_data(
content_text="<|im_start|>A small red cube on a white table.<|im_end|>"
),
"modalities": ["image"],
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"guidance_scale": 0.0,
"seed": 42,
},
}
responses = openai_client.send_diffusion_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
assert len(responses) == ROUTE_STRESS_REQUESTS
assert all(response.success for response in responses)
assert all(response.images is not None and response.images[0].size == (512, 512) for response in responses)
@@ -0,0 +1,137 @@
"""
Core-model CI guard for Qwen3-Omni multi-replica stage-pool routing on 4 GPUs.
"""
import os
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
from tests.helpers.runtime import OmniResponse, OmniServerParams, dummy_messages_from_mix_data
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
MULTI_REPLICA_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe_multi_replicas_4gpu.yaml")
ROUTE_STRESS_REQUESTS = 6
MIXED_MODAL_REQUESTS = 4
test_params = [
OmniServerParams(
model=MODEL,
stage_config_path=MULTI_REPLICA_DEPLOY,
server_args=["--disable-log-stats"],
)
]
def _system_prompt() -> dict[str, object]:
return {
"role": "system",
"content": [
{
"type": "text",
"text": (
"You are Qwen, a virtual human developed by the Qwen Team, "
"Alibaba Group, capable of perceiving auditory and visual inputs, "
"as well as generating text and speech."
),
}
],
}
def _text_messages() -> list[dict[str, object]]:
return dummy_messages_from_mix_data(
system_prompt=_system_prompt(),
content_text="What is the capital of China? Answer in one short sentence.",
)
def _mixed_messages() -> list[dict[str, object]]:
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
return dummy_messages_from_mix_data(
system_prompt=_system_prompt(),
video_data_url=video_data_url,
image_data_url=image_data_url,
audio_data_url=audio_data_url,
content_text="What is recited in the audio? What is in this image? Describe the video briefly.",
)
def _assert_batch_size(responses: list[OmniResponse], expected: int) -> None:
assert len(responses) == expected, f"Expected {expected} responses, got {len(responses)}"
assert all(resp.success for resp in responses), "At least one request failed"
assert all(resp.e2e_latency is not None and resp.e2e_latency > 0 for resp in responses), "Missing request latency"
def _assert_text_outputs(responses: list[OmniResponse]) -> None:
assert all(resp.text_content is not None for resp in responses), "Missing text output"
assert all(resp.audio_bytes is None for resp in responses), "Text-only request unexpectedly produced audio"
def _assert_audio_outputs(responses: list[OmniResponse], *, expect_text: bool) -> None:
assert all(resp.audio_bytes is not None and len(resp.audio_bytes) > 128 for resp in responses), (
"Missing audio output"
)
if expect_text:
assert all(resp.text_content is not None for resp in responses), "Missing text output"
else:
assert all(not (resp.text_content or "").strip() for resp in responses), (
"Audio-only request unexpectedly produced text"
)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_only_batch_uses_multi_replica_talker(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": _text_messages(),
"stream": False,
"modalities": ["text"],
}
responses = openai_client.send_omni_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
_assert_batch_size(responses, ROUTE_STRESS_REQUESTS)
_assert_text_outputs(responses)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": _text_messages(),
"stream": True,
"modalities": ["audio"],
}
responses = openai_client.send_omni_request(request_config, request_num=ROUTE_STRESS_REQUESTS)
_assert_batch_size(responses, ROUTE_STRESS_REQUESTS)
_assert_audio_outputs(responses, expect_text=False)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_mixed_modal_stream_batch_generates_text_and_audio(omni_server, openai_client) -> None:
request_config = {
"model": omni_server.model,
"messages": _mixed_messages(),
"stream": True,
}
responses = openai_client.send_omni_request(request_config, request_num=MIXED_MODAL_REQUESTS)
_assert_batch_size(responses, MIXED_MODAL_REQUESTS)
_assert_audio_outputs(responses, expect_text=True)
+1 -1
View File
@@ -59,7 +59,7 @@ def test_build_add_request_message_preserves_additional_information(mocker: Mock
assert request.additional_information is not None
assert request.additional_information.entries["text"].list_data == ["hello world"]
assert request.additional_information.entries["speaker"].list_data == ["vivian"]
output_processor.add_request.assert_called_once()
output_processor.add_request.assert_not_called()
def test_build_add_request_message_with_resumable_streaming(mocker: MockerFixture):
+470 -264
View File
@@ -1,15 +1,138 @@
import importlib
import os
import threading
import time
import types
import pytest
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.engine.stage_init_utils import (
LogicalStageInitPlan,
ReplicaInitPlan,
build_stage0_input_processor,
compute_replica_layout,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_llm_metadata(
stage_id: int,
*,
replica_id: int = 0,
final_output: bool = False,
final_output_type: str | None = None,
is_comprehension: bool = False,
):
return types.SimpleNamespace(
stage_id=stage_id,
stage_type="llm",
runtime_cfg={},
prompt_expand_func=None,
final_output=final_output,
final_output_type=final_output_type,
default_sampling_params=types.SimpleNamespace(name=f"sp-{stage_id}-{replica_id}"),
custom_process_input_func=None,
engine_input_source=[] if stage_id == 0 else [stage_id - 1],
engine_output_type="token_ids",
replica_id=replica_id,
is_comprehension=is_comprehension,
)
def _make_diffusion_metadata(stage_id: int, *, replica_id: int = 0, final_output_type: str = "image"):
return types.SimpleNamespace(
stage_id=stage_id,
stage_type="diffusion",
runtime_cfg={"devices": str(replica_id)},
prompt_expand_func=None,
final_output=True,
final_output_type=final_output_type,
default_sampling_params=types.SimpleNamespace(name=f"dsp-{stage_id}-{replica_id}"),
custom_process_input_func=None,
engine_input_source=[],
cfg_kv_collect_func=None,
replica_id=replica_id,
)
def _make_llm_plan(
stage_idx: int,
*,
configured_stage_id: int,
vllm_config: object,
num_replicas: int = 1,
final_output: bool = False,
final_output_type: str | None = None,
is_comprehension: bool = False,
):
replicas: list[ReplicaInitPlan] = []
for replica_id in range(num_replicas):
stage_cfg = types.SimpleNamespace(
stage_id=configured_stage_id,
stage_type="llm",
runtime=types.SimpleNamespace(devices=str(replica_id)),
engine_args={},
)
replicas.append(
ReplicaInitPlan(
replica_id=replica_id,
num_replicas=num_replicas,
launch_mode="local",
stage_cfg=stage_cfg,
metadata=_make_llm_metadata(
configured_stage_id,
replica_id=replica_id,
final_output=final_output,
final_output_type=final_output_type,
is_comprehension=is_comprehension and replica_id == 0,
),
stage_connector_spec={},
omni_kv_connector=(None, None, None),
stage_vllm_config=vllm_config,
executor_class=object,
)
)
return LogicalStageInitPlan(
stage_idx=stage_idx,
configured_stage_id=configured_stage_id,
replicas=replicas,
)
def _make_diffusion_plan(
stage_idx: int,
*,
configured_stage_id: int,
num_replicas: int = 1,
):
replicas: list[ReplicaInitPlan] = []
for replica_id in range(num_replicas):
stage_cfg = types.SimpleNamespace(
stage_id=configured_stage_id,
stage_type="diffusion",
runtime=types.SimpleNamespace(devices=str(replica_id)),
engine_args={},
)
replicas.append(
ReplicaInitPlan(
replica_id=replica_id,
num_replicas=num_replicas,
launch_mode="local",
stage_cfg=stage_cfg,
metadata=_make_diffusion_metadata(configured_stage_id, replica_id=replica_id),
stage_connector_spec={},
omni_kv_connector=(None, None, None),
)
)
return LogicalStageInitPlan(
stage_idx=stage_idx,
configured_stage_id=configured_stage_id,
replicas=replicas,
)
def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
"""Regression test for forward references in make_async_mp_client."""
import vllm_omni.engine.stage_engine_core_client as client_mod
@@ -21,15 +144,279 @@ def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
)
def test_initialize_stages_restores_device_visibility_after_diffusion_init(monkeypatch):
"""Regression test for stage device env leakage across stage init.
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
stage_cfg = types.SimpleNamespace(
stage_id=0,
stage_type="diffusion",
engine_args={"parallel_config": {"tensor_parallel_size": 2}},
runtime={"devices": "0,1,2,3", "num_replicas": 2},
)
Diffusion init mutates process-level CUDA visibility. Ensure AsyncOmniEngine
restores the previous value after diffusion stage setup.
"""
replicas_per_stage, replica_devices_map = compute_replica_layout([stage_cfg])
assert replicas_per_stage == [2]
assert replica_devices_map == {0: ["0,1", "2,3"]}
def test_collect_initialized_clients_for_cleanup_deduplicates_clients():
shared = types.SimpleNamespace(name="shared")
extra = types.SimpleNamespace(name="extra")
cleanup_clients = AsyncOmniEngine._collect_initialized_clients_for_cleanup(
stage_pools=[types.SimpleNamespace(clients=[shared, None])],
initialized_clients_by_stage={0: [shared], 1: [extra]},
)
assert cleanup_clients == [shared, extra]
def test_initialize_stages_rejects_non_diffusion_replicas_in_single_stage_mode():
engine = object.__new__(AsyncOmniEngine)
engine.single_stage_mode = True
engine.stage_configs = [types.SimpleNamespace(stage_id=0, runtime={"num_replicas": 2})]
with pytest.raises(
ValueError,
match="single_stage_mode only supports num_replicas > 1 for diffusion stages",
):
engine._validate_single_stage_mode_replica_constraints()
def test_initialize_diffusion_replica_restores_device_visibility_after_local_init(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
from vllm_omni.platforms import current_omni_platform
engine = object.__new__(AsyncOmniEngine)
engine.model = "dummy-model"
engine.num_stages = 1
engine.diffusion_batch_size = 1
engine.single_stage_mode = False
engine._omni_master_server = None
engine.stage_configs = []
plan = _make_diffusion_plan(0, configured_stage_id=0).replicas[0]
env_var = current_omni_platform.device_control_env_var
old_env = os.environ.get(env_var)
os.environ[env_var] = "0,1"
def _fake_setup_stage_devices(_stage_id, _runtime_cfg):
current_omni_platform.set_device_control_env_var("1")
monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices)
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: types.SimpleNamespace())
try:
engine._initialize_diffusion_replica(plan, stage_init_timeout=1, stage_launch_lock=threading.Lock())
assert os.environ.get(env_var) == "0,1"
finally:
if old_env is None:
os.environ.pop(env_var, None)
else:
os.environ[env_var] = old_env
def test_initialize_diffusion_replica_passes_stage_init_timeout_and_inline_flag(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
engine = object.__new__(AsyncOmniEngine)
engine.model = "dummy-model"
engine.num_stages = 1
engine.diffusion_batch_size = 4
engine.single_stage_mode = False
engine._omni_master_server = None
engine.stage_configs = []
plan = _make_diffusion_plan(0, configured_stage_id=0).replicas[0]
captured: dict[str, object] = {}
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
def _capture_initialize_diffusion_stage(
stage_id, _model, _stage_cfg, _metadata, *, stage_init_timeout, batch_size, use_inline
):
captured["stage_id"] = stage_id
captured["stage_init_timeout"] = stage_init_timeout
captured["batch_size"] = batch_size
captured["use_inline"] = use_inline
return types.SimpleNamespace()
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", _capture_initialize_diffusion_stage)
engine._initialize_diffusion_replica(plan, stage_init_timeout=302, stage_launch_lock=threading.Lock())
assert captured == {
"stage_id": 0,
"stage_init_timeout": 302,
"batch_size": 4,
"use_inline": True,
}
def test_initialize_stages_exposes_logical_stage_views_and_builds_top_level_input_processor(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
engine = object.__new__(AsyncOmniEngine)
engine.model = "dummy-model"
engine.config_path = "dummy-config"
engine.num_stages = 2
engine.async_chunk = False
engine.diffusion_batch_size = 1
engine.single_stage_mode = False
engine._single_stage_id_filter = None
engine._omni_master_server = None
engine.stage_configs = [types.SimpleNamespace(), types.SimpleNamespace()]
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
cfg1 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
stage_plans = [
_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2, is_comprehension=True),
_make_llm_plan(1, configured_stage_id=1, vllm_config=cfg1, final_output=True),
]
stage0_client_r0 = types.SimpleNamespace(
stage_type="llm",
is_comprehension=True,
final_output=False,
final_output_type=None,
default_sampling_params=types.SimpleNamespace(name="sp0"),
)
stage0_client_r1 = types.SimpleNamespace(
stage_type="llm",
is_comprehension=False,
final_output=False,
final_output_type=None,
default_sampling_params=types.SimpleNamespace(name="sp0r1"),
)
stage1_client_r0 = types.SimpleNamespace(
stage_type="llm",
is_comprehension=False,
final_output=True,
final_output_type=None,
default_sampling_params=types.SimpleNamespace(name="sp1"),
)
initialized_clients = {
0: [stage0_client_r0, stage0_client_r1],
1: [stage1_client_r0],
}
stage0_output_processor = object()
stage1_output_processor = object()
top_level_input_processor = object()
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2, 1], {}))
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
monkeypatch.setattr(engine, "_initialize_stage_replicas", lambda *_: initialized_clients)
monkeypatch.setattr(
engine_mod,
"build_llm_stage_output_processor",
lambda plan, _cfg: stage0_output_processor if plan.stage_idx == 0 else stage1_output_processor,
)
monkeypatch.setattr(engine_mod, "build_stage0_input_processor", lambda _cfg: top_level_input_processor)
engine._initialize_stages(stage_init_timeout=1)
assert len(engine.stage_pools) == 2
assert engine.input_processor is top_level_input_processor
assert engine.stage_clients == [stage0_client_r0, stage1_client_r0]
assert engine.stage_vllm_configs == [cfg0, cfg1]
assert engine.output_processors == [stage0_output_processor, stage1_output_processor]
assert engine.default_sampling_params_list == [
stage0_client_r0.default_sampling_params,
stage1_client_r0.default_sampling_params,
]
assert engine.stage_metadata == [
{"final_output": False, "final_output_type": None, "stage_type": "llm"},
{"final_output": True, "final_output_type": None, "stage_type": "llm"},
]
def test_build_logical_stage_init_plans_applies_replica_device_splits(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
engine = object.__new__(AsyncOmniEngine)
engine.model = "dummy-model"
engine.async_chunk = False
engine.single_stage_mode = False
engine._single_stage_id_filter = None
engine.stage_configs = [
types.SimpleNamespace(stage_id=0, stage_type="llm", engine_args={}, runtime=types.SimpleNamespace(devices="0")),
types.SimpleNamespace(
stage_id=1, stage_type="llm", engine_args={}, runtime=types.SimpleNamespace(devices="1,2,3")
),
]
metadata_by_stage = {
0: _make_llm_metadata(0),
1: _make_llm_metadata(1),
}
monkeypatch.setattr(
engine_mod,
"extract_stage_metadata",
lambda cfg: types.SimpleNamespace(**metadata_by_stage[cfg.stage_id].__dict__),
)
monkeypatch.setattr(engine_mod, "get_stage_connector_spec", lambda **_: {})
monkeypatch.setattr(engine_mod, "resolve_omni_kv_config_for_stage", lambda *_: (None, None, None))
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
monkeypatch.setattr(
engine_mod,
"build_vllm_config",
lambda stage_cfg, *_args, **_kwargs: (types.SimpleNamespace(tag=f"cfg-{stage_cfg.stage_id}"), object),
)
stage_plans, prompt_expand_func = engine._build_logical_stage_init_plans(
omni_transfer_config=None,
replicas_per_stage=[1, 3],
replica_devices_map={1: ["1", "2", "3"]},
)
assert prompt_expand_func is None
assert [plan.configured_stage_id for plan in stage_plans] == [0, 1]
assert [replica.stage_cfg.runtime.devices for replica in stage_plans[1].replicas] == ["1", "2", "3"]
assert [replica.replica_id for replica in stage_plans[1].replicas] == [0, 1, 2]
assert all(replica.num_replicas == 3 for replica in stage_plans[1].replicas)
def test_initialize_stage_replicas_collects_results_by_stage_and_replica_id(monkeypatch):
engine = object.__new__(AsyncOmniEngine)
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
cfg1 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
stage_plans = [
_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2),
_make_llm_plan(1, configured_stage_id=1, vllm_config=cfg1, num_replicas=2),
]
clients = {
(0, 0): types.SimpleNamespace(name="stage0-replica0"),
(0, 1): types.SimpleNamespace(name="stage0-replica1"),
(1, 0): types.SimpleNamespace(name="stage1-replica0"),
(1, 1): types.SimpleNamespace(name="stage1-replica1"),
}
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
time.sleep(0.02 * (3 - plan.metadata.stage_id - plan.replica_id))
return clients[(plan.metadata.stage_id, plan.replica_id)]
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
initialized_clients = engine._initialize_stage_replicas(stage_plans, stage_init_timeout=123)
assert initialized_clients == {
0: [clients[(0, 0)], clients[(0, 1)]],
1: [clients[(1, 0)], clients[(1, 1)]],
}
def test_initialize_stages_cleans_up_successful_replicas_after_partial_multi_replica_failure(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
engine = object.__new__(AsyncOmniEngine)
engine.model = "dummy-model"
engine.config_path = "dummy-config"
@@ -39,158 +426,111 @@ def test_initialize_stages_restores_device_visibility_after_diffusion_init(monke
engine.single_stage_mode = False
engine._single_stage_id_filter = None
engine._omni_master_server = None
engine.stage_configs = [types.SimpleNamespace(stage_id=0, stage_type="diffusion")]
engine.stage_configs = [types.SimpleNamespace()]
env_var = current_omni_platform.device_control_env_var
old_env = os.environ.get(env_var)
os.environ[env_var] = "0,1"
diffusion_client = types.SimpleNamespace(is_comprehension=False)
metadata = types.SimpleNamespace(
stage_id=0,
stage_type="diffusion",
runtime_cfg={"devices": "1"},
prompt_expand_func=None,
)
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
stage_plans = [_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2)]
initialized_client = types.SimpleNamespace(shutdown=lambda: None)
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
monkeypatch.setattr(engine_mod, "extract_stage_metadata", lambda _cfg: metadata)
monkeypatch.setattr(engine_mod, "get_stage_connector_spec", lambda **_: {})
monkeypatch.setattr(engine_mod, "resolve_omni_kv_config_for_stage", lambda *_: (None, None, None))
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2], {}))
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
def _fake_setup_stage_devices(_stage_id, _runtime_cfg):
# Simulate diffusion setup mutating process-global visibility.
current_omni_platform.set_device_control_env_var("1")
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
if plan.replica_id == 0:
return initialized_client
time.sleep(0.05)
raise RuntimeError("replica launch failed")
monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices)
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: diffusion_client)
monkeypatch.setattr(
engine_mod,
"finalize_initialized_stages",
lambda stage_clients, _input_processor: (
stage_clients,
[types.SimpleNamespace()],
[{"final_output_type": "image"}],
),
)
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
try:
captured_cleanup: list[list[object]] = []
def _capture_shutdown(clients):
captured_cleanup.append(list(clients))
monkeypatch.setattr(engine, "_shutdown_initialized_clients", _capture_shutdown)
with pytest.raises(RuntimeError, match="replica launch failed"):
engine._initialize_stages(stage_init_timeout=1)
assert os.environ.get(env_var) == "0,1"
finally:
if old_env is None:
os.environ.pop(env_var, None)
else:
os.environ[env_var] = old_env
assert captured_cleanup == [[initialized_client]]
def test_initialize_stages_passes_stage_init_timeout_to_diffusion_handshake(monkeypatch):
"""Regression test for stage_init_timeout passing to complete_diffusion_handshake
in the diffusion stage path.
"""
import vllm_omni.diffusion.data as diffusion_data_mod
import vllm_omni.diffusion.stage_diffusion_client as client_mod
def test_initialize_stages_cleans_up_late_successful_replicas_after_early_multi_replica_failure(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
from vllm_omni.platforms import current_omni_platform
engine = object.__new__(AsyncOmniEngine)
engine.log_stats = False
engine.model = "dummy-model"
engine.config_path = "dummy-config"
engine.num_stages = 2
engine.num_stages = 1
engine.async_chunk = False
engine.diffusion_batch_size = 1
engine.single_stage_mode = False
engine._single_stage_id_filter = None
engine._omni_master_server = None
engine.stage_configs = [types.SimpleNamespace(stage_id=0, stage_type="diffusion", engine_args={})]
engine.stage_configs = [types.SimpleNamespace()]
metadata = types.SimpleNamespace(
stage_id=0,
stage_type="diffusion",
runtime_cfg={"devices": "0"},
prompt_expand_func=None,
final_output=True,
final_output_type="image",
default_sampling_params=None,
custom_process_input_func=None,
engine_input_source=None,
cfg_kv_collect_func=None,
)
captured_timeout = None
device_env_var = current_omni_platform.device_control_env_var
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
stage_plans = [_make_llm_plan(0, configured_stage_id=0, vllm_config=cfg0, num_replicas=2)]
initialized_client = types.SimpleNamespace(shutdown=lambda: None)
replica1_started = threading.Event()
monkeypatch.setattr(engine_mod, "prepare_engine_environment", lambda: None)
monkeypatch.setattr(engine_mod, "load_omni_transfer_config_for_model", lambda *_: None)
monkeypatch.setattr(engine_mod, "extract_stage_metadata", lambda _cfg: metadata)
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
monkeypatch.setattr(
engine_mod,
"finalize_initialized_stages",
lambda stage_clients, _input_processor: (
stage_clients,
[types.SimpleNamespace()],
[{"final_output_type": "image"}],
),
)
monkeypatch.setattr(
diffusion_data_mod.OmniDiffusionConfig,
"from_kwargs",
classmethod(lambda cls, **kwargs: types.SimpleNamespace(parallel_config=types.SimpleNamespace(world_size=1))),
)
monkeypatch.setattr(
client_mod,
"spawn_diffusion_proc",
lambda model, od_cfg: (object(), "ipc://handshake", "ipc://request", "ipc://response"),
)
monkeypatch.setattr(engine_mod, "compute_replica_layout", lambda _cfgs: ([2], {}))
monkeypatch.setattr(engine, "_build_logical_stage_init_plans", lambda *_: (stage_plans, None))
def _capture_handshake_timeout(proc, handshake_address, handshake_timeout):
nonlocal captured_timeout
captured_timeout = handshake_timeout
def _initialize_replica(plan, _stage_init_timeout, _stage_launch_lock):
if plan.replica_id == 0:
assert replica1_started.wait(timeout=1)
raise RuntimeError("replica launch failed")
replica1_started.set()
time.sleep(0.05)
return initialized_client
monkeypatch.setattr(client_mod, "complete_diffusion_handshake", _capture_handshake_timeout)
monkeypatch.setattr(
client_mod.zmq,
"Context",
lambda: types.SimpleNamespace(socket=lambda _: types.SimpleNamespace(connect=lambda _: None)),
)
monkeypatch.setattr(engine, "_initialize_replica", _initialize_replica)
try:
engine._initialize_stages(stage_init_timeout=302)
finally:
if prev_device_env is None:
os.environ.pop(device_env_var, None)
else:
os.environ[device_env_var] = prev_device_env
captured_cleanup: list[list[object]] = []
assert captured_timeout == 302
def _capture_shutdown(clients):
captured_cleanup.append(list(clients))
monkeypatch.setattr(engine, "_shutdown_initialized_clients", _capture_shutdown)
with pytest.raises(RuntimeError, match="replica launch failed"):
engine._initialize_stages(stage_init_timeout=1)
assert captured_cleanup == [[initialized_client]]
def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(monkeypatch):
"""Regression test for stage_init_timeout reaching complete_stage_handshake
in the LLM stage path.
"""
def test_initialize_llm_replica_passes_stage_init_timeout_to_complete_stage_handshake(monkeypatch):
import vllm_omni.engine.async_omni_engine as engine_mod
from vllm_omni.platforms import current_omni_platform
engine = object.__new__(AsyncOmniEngine)
engine.log_stats = False
engine.model = "dummy-model"
engine.single_stage_mode = False
engine._omni_master_server = None
engine.stage_configs = []
metadata = types.SimpleNamespace(stage_id=0, runtime_cfg={"devices": "0"})
fake_vllm_config = types.SimpleNamespace()
fake_addresses = types.SimpleNamespace()
fake_addresses = types.SimpleNamespace(inputs=["in"], outputs=["out"], frontend_stats_publish_address=None)
fake_proc = types.SimpleNamespace()
captured_timeout: int | None = None
captured_timeout = None
plan = ReplicaInitPlan(
replica_id=0,
num_replicas=1,
launch_mode="local",
stage_cfg=types.SimpleNamespace(engine_args={}, runtime=types.SimpleNamespace(devices="0")),
metadata=types.SimpleNamespace(stage_id=0, runtime_cfg={"devices": "0"}),
stage_connector_spec={},
omni_kv_connector=(None, None, None),
stage_vllm_config=fake_vllm_config,
executor_class=object,
)
device_env_var = current_omni_platform.device_control_env_var
prev_device_env = os.environ.get(device_env_var)
@@ -198,28 +538,22 @@ def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
monkeypatch.setattr(engine_mod, "build_vllm_config", lambda *_, **__: (fake_vllm_config, object))
monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: [])
monkeypatch.setattr(
engine_mod,
"spawn_stage_core",
lambda **_: (fake_addresses, fake_proc, "ipc://handshake"),
)
monkeypatch.setattr(engine_mod, "spawn_stage_core", lambda **_: (fake_addresses, fake_proc, "ipc://handshake"))
def _capture_stage_timeout(_proc, _handshake_addr, _addresses, _vllm_cfg, handshake_timeout):
nonlocal captured_timeout
captured_timeout = handshake_timeout
monkeypatch.setattr(engine_mod, "complete_stage_handshake", _capture_stage_timeout)
monkeypatch.setattr(
engine_mod.StageEngineCoreClientBase,
"make_async_mp_client",
staticmethod(lambda **_: types.SimpleNamespace(shutdown=lambda: None)),
)
try:
engine._launch_llm_stage(
stage_cfg=types.SimpleNamespace(engine_args={}),
metadata=metadata,
stage_connector_spec={},
stage_init_timeout=302,
llm_stage_launch_lock=threading.Lock(),
)
engine._initialize_llm_replica(plan, 302, threading.Lock())
finally:
if prev_device_env is None:
os.environ.pop(device_env_var, None)
@@ -229,155 +563,27 @@ def test_launch_llm_stage_passes_stage_init_timeout_to_complete_stage_handshake(
assert captured_timeout == 302
def test_launch_llm_stage_releases_launch_lock_before_complete_stage_handshake(monkeypatch):
"""Regression test for parallel LLM stage startup during handshake wait."""
import vllm_omni.engine.async_omni_engine as engine_mod
from vllm_omni.platforms import current_omni_platform
engine = object.__new__(AsyncOmniEngine)
engine.log_stats = False
engine.model = "dummy-model"
engine.single_stage_mode = False
engine._omni_master_server = None
engine.stage_configs = []
fake_vllm_config = types.SimpleNamespace()
fake_addresses = types.SimpleNamespace()
shared_launch_lock = threading.Lock()
counter_lock = threading.Lock()
first_handshake_started = threading.Event()
second_stage_spawned = threading.Event()
allow_first_handshake_to_finish = threading.Event()
launch_errors: list[BaseException] = []
spawn_count = 0
device_env_var = current_omni_platform.device_control_env_var
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
monkeypatch.setattr(engine_mod, "build_vllm_config", lambda *_, **__: (fake_vllm_config, object))
monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: [])
def _spawn_stage_core(**_):
nonlocal spawn_count
with counter_lock:
spawn_count += 1
call_idx = spawn_count
if call_idx == 2:
second_stage_spawned.set()
return fake_addresses, types.SimpleNamespace(), f"ipc://handshake-{call_idx}"
def _complete_stage_handshake(_proc, handshake_address, _addresses, _vllm_cfg, _timeout):
if handshake_address == "ipc://handshake-1":
first_handshake_started.set()
assert second_stage_spawned.wait(timeout=1), (
"second stage did not reach spawn_stage_core while first stage waited in handshake"
)
assert allow_first_handshake_to_finish.wait(timeout=1), (
"second stage did not enter handshake while first stage was still waiting"
)
else:
allow_first_handshake_to_finish.set()
monkeypatch.setattr(engine_mod, "spawn_stage_core", _spawn_stage_core)
monkeypatch.setattr(engine_mod, "complete_stage_handshake", _complete_stage_handshake)
def _launch_stage(stage_id: int) -> None:
metadata = types.SimpleNamespace(stage_id=stage_id, runtime_cfg={"devices": str(stage_id)})
try:
engine._launch_llm_stage(
stage_cfg=types.SimpleNamespace(engine_args={}),
metadata=metadata,
stage_connector_spec={},
stage_init_timeout=302,
llm_stage_launch_lock=shared_launch_lock,
)
except BaseException as exc: # pragma: no cover - surfaced through assertion below
launch_errors.append(exc)
try:
first_thread = threading.Thread(target=_launch_stage, args=(0,))
first_thread.start()
assert first_handshake_started.wait(timeout=1), "first stage never entered handshake"
second_thread = threading.Thread(target=_launch_stage, args=(1,))
second_thread.start()
first_thread.join(timeout=3)
second_thread.join(timeout=3)
finally:
if prev_device_env is None:
os.environ.pop(device_env_var, None)
else:
os.environ[device_env_var] = prev_device_env
assert not first_thread.is_alive()
assert not second_thread.is_alive()
assert second_stage_spawned.is_set()
assert not launch_errors
def test_attach_llm_stage_uses_omni_input_preprocessor(monkeypatch):
"""Regression test for GLM-Image t2i preprocessing path.
Stage-0 InputProcessor must use OmniInputPreprocessor so text prompts with
mm_processor_kwargs go through multimodal preprocessing.
"""
import vllm_omni.engine.async_omni_engine as engine_mod
class DummyStageEngineCoreClient:
def __init__(self, **kwargs):
self.kwargs = kwargs
def shutdown(self):
return None
class DummyOutputProcessor:
def __init__(self, **kwargs):
self.kwargs = kwargs
def test_build_stage0_input_processor_uses_omni_input_preprocessor(monkeypatch):
import vllm_omni.engine.stage_init_utils as init_mod
class DummyInputProcessor:
def __init__(self, vllm_config):
self.vllm_config = vllm_config
self.renderer = object()
self.input_preprocessor = object()
self.input_preprocessor = None
class DummyOmniInputPreprocessor:
def __init__(self, vllm_config, renderer=None):
self.vllm_config = vllm_config
self.renderer = renderer
monkeypatch.setattr(
engine_mod.StageEngineCoreClientBase,
"make_async_mp_client",
staticmethod(lambda **kwargs: DummyStageEngineCoreClient(**kwargs)),
)
monkeypatch.setattr(engine_mod, "MultimodalOutputProcessor", DummyOutputProcessor)
monkeypatch.setattr(engine_mod, "InputProcessor", DummyInputProcessor)
monkeypatch.setattr(engine_mod, "OmniInputPreprocessor", DummyOmniInputPreprocessor)
monkeypatch.setattr(init_mod, "InputProcessor", DummyInputProcessor)
monkeypatch.setattr(init_mod, "OmniInputPreprocessor", DummyOmniInputPreprocessor)
started = types.SimpleNamespace(
stage_id=0,
metadata=types.SimpleNamespace(stage_id=0, engine_output_type="token_ids"),
vllm_config=types.SimpleNamespace(model_config=types.SimpleNamespace(skip_tokenizer_init=True)),
executor_class=object,
engine_manager=object(),
coordinator=object(),
proc=None,
addresses=types.SimpleNamespace(
inputs=["inproc://input"],
outputs=["inproc://output"],
frontend_stats_publish_address=None,
),
input_processor = build_stage0_input_processor(
types.SimpleNamespace(model_config=types.SimpleNamespace(try_get_generation_config=lambda: {}))
)
engine = object.__new__(AsyncOmniEngine)
_stage_client, _out_proc, _vllm_cfg, input_processor = engine._attach_llm_stage(started)
assert input_processor is not None
assert isinstance(input_processor.input_preprocessor, DummyOmniInputPreprocessor)
assert input_processor.input_preprocessor.renderer is input_processor.renderer
@@ -1,7 +1,6 @@
import pytest
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -25,18 +24,6 @@ def test_register_companion_and_cleanup():
assert tracker.get_companion_request_ids("req1") == {}
def test_attach_cfg_request_ids_clones_diffusion_params():
tracker = CfgCompanionTracker()
tracker.register_companion("req1", "cfg_text", "req1__cfg_text")
params = OmniDiffusionSamplingParams()
updated = tracker.attach_cfg_request_ids("req1", params)
assert updated is not params
assert params.cfg_kv_request_ids is None
assert updated.cfg_kv_request_ids == {"cfg_text": "req1__cfg_text"}
def test_abort_parent_expands_to_companions_and_cleans_up_deferred_parent():
tracker = CfgCompanionTracker()
tracker.register_companion("req1", "cfg_text", "req1__cfg_text")
+509 -15
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import concurrent.futures
import logging
import queue
import threading
import time
@@ -16,8 +17,9 @@ from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.core_client import AsyncMPClient
from vllm_omni.engine.orchestrator import Orchestrator
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -73,7 +75,7 @@ class FakeStageClient:
def set_engine_outputs(self, outputs) -> None:
return None
def process_engine_inputs(self, stage_list, prompt=None, streaming_context=None):
def process_engine_inputs(self, source_outputs, prompt=None, streaming_context=None):
return list(self.next_inputs)
async def abort_requests_async(self, request_ids: list[str]) -> None:
@@ -90,12 +92,33 @@ class FakeStageClient:
self._diffusion_outputs.put_nowait(output)
class FakeCollectiveRpcStageClient(FakeStageClient):
def __init__(self, *args, rpc_result: Any = None, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.rpc_result = rpc_result
self.collective_rpc_calls: list[tuple[str, float | None, tuple[Any, ...], dict[str, Any]]] = []
async def collective_rpc_async(
self,
*,
method: str,
timeout: float | None = None,
args: tuple[Any, ...] = (),
kwargs: dict[str, Any] | None = None,
) -> Any:
normalized_kwargs = dict(kwargs or {})
self.collective_rpc_calls.append((method, timeout, args, normalized_kwargs))
return self.rpc_result
class FakeOutputProcessor:
def __init__(self, *, request_outputs: list[object] | None = None) -> None:
self.request_outputs = list(request_outputs or [])
self.add_request_calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
self.abort_calls: list[list[str]] = []
def add_request(self, *_args, **_kwargs) -> None:
def add_request(self, *args, **kwargs) -> None:
self.add_request_calls.append((args, kwargs))
return None
def process_outputs(self, *_args, **_kwargs):
@@ -185,17 +208,60 @@ def _build_request_output(
)
def _build_stage_pools(
stage_clients: list[list[FakeStageClient]],
*,
output_processors: list[FakeOutputProcessor] | None = None,
stage_vllm_configs: list[object] | None = None,
) -> list[StagePool]:
"""Build StagePool list from per-stage replica lists.
``stage_clients[i]`` is the list of FakeStageClient replicas for stage i.
"""
num_stages = len(stage_clients)
if output_processors is None:
output_processors = [FakeOutputProcessor() for _ in stage_clients]
if stage_vllm_configs is None:
stage_vllm_configs = [SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)) for _ in stage_clients]
pools: list[StagePool] = []
for stage_id in range(num_stages):
clients = stage_clients[stage_id]
if clients[0].stage_type == "diffusion":
pools.append(StagePool(stage_id, clients[0]))
else:
pools.append(
StagePool(
stage_id,
clients,
output_processor=output_processors[stage_id],
stage_vllm_config=stage_vllm_configs[stage_id],
)
)
return pools
def _build_harness(
stage_clients: list[object],
*,
output_processors: list[object] | None = None,
stage_vllm_configs: list[object] | None = None,
async_chunk: bool = False,
stage_pools: list[StagePool] | None = None,
) -> OrchestratorFixture:
if output_processors is None:
output_processors = [FakeOutputProcessor() for _ in stage_clients]
if stage_vllm_configs is None:
stage_vllm_configs = [SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)) for _ in stage_clients]
"""Build an Orchestrator test harness.
Accepts either pre-built ``stage_pools`` or flat lists of single-replica
clients/processors.
"""
if stage_pools is None:
# Wrap flat lists into per-stage single-replica lists.
nested_clients = [[c] for c in stage_clients]
stage_pools = _build_stage_pools(
nested_clients,
output_processors=output_processors,
stage_vllm_configs=stage_vllm_configs,
)
ready_future: concurrent.futures.Future[tuple[Orchestrator, janus.Queue, janus.Queue, janus.Queue]] = (
concurrent.futures.Future()
@@ -214,9 +280,7 @@ def _build_harness(
request_async_queue=request_queue.async_q,
output_async_queue=output_queue.async_q,
rpc_async_queue=rpc_queue.async_q,
stage_clients=stage_clients,
output_processors=output_processors,
stage_vllm_configs=stage_vllm_configs,
stage_pools=stage_pools,
async_chunk=async_chunk,
)
ready_future.set_result((orchestrator, request_queue, output_queue, rpc_queue))
@@ -283,6 +347,18 @@ async def _get_output_message(orchestrator_fixture: OrchestratorFixture, *, time
return msg
async def _get_rpc_message(orchestrator_fixture: OrchestratorFixture, *, timeout: float = 2.0) -> dict:
deadline = time.monotonic() + timeout
rpc_sync_q = orchestrator_fixture.queues[2].sync_q
while True:
if time.monotonic() >= deadline:
raise AssertionError("Timed out waiting for orchestrator rpc output")
try:
return rpc_sync_q.get_nowait()
except queue.Empty:
await asyncio.sleep(0.01)
async def _enqueue_add_request(
orchestrator_fixture: OrchestratorFixture,
*,
@@ -387,6 +463,11 @@ def orchestrator_factory():
q.close()
# ---------------------------------------------------------------------------
# Existing single-replica tests (adapted to StagePool interface)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_run_two_stage_llm(orchestrator_factory) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=False)
@@ -600,12 +681,425 @@ async def test_run_abort(orchestrator_factory) -> None:
await _wait_for(lambda: len(stages[0].add_request_calls) == 1)
await _enqueue_abort_request(orchestrator_fixture, ["req-abort"])
await _wait_for(lambda: all(stage.abort_calls for stage in stages))
await _wait_for(lambda: bool(stages[0].abort_calls))
for stage in stages:
assert stage.abort_calls == [["req-abort"]]
for proc in processors:
assert proc.abort_calls == [["req-abort"]]
assert stages[0].abort_calls == [["req-abort"]]
assert stages[1].abort_calls == []
assert "req-abort" not in orchestrator_fixture.orchestrator.request_states
finally:
await _shutdown_orchestrator(orchestrator_fixture)
# ---------------------------------------------------------------------------
# Multi-replica tests
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_multi_replica_round_robin_distribution(orchestrator_factory) -> None:
"""Two replicas at stage-0, single replica at stage-1.
Send two requests — they should land on different stage-0 replicas
(round-robin), then both forward to the single stage-1 replica.
"""
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
stage1 = FakeStageClient(
stage_type="llm",
final_output=True,
next_inputs=[{"prompt_token_ids": [7, 8]}],
)
proc0 = FakeOutputProcessor(request_outputs=[_build_request_output("req-0", token_ids=[3], finished=True)])
proc1 = FakeOutputProcessor(request_outputs=[_build_request_output("req-0", token_ids=[10], finished=True)])
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
stage_pools = _build_stage_pools(
[[stage0_r0, stage0_r1], [stage1]],
output_processors=[proc0, proc1],
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
)
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
try:
# Request 0 → should land on replica 0 (RR starts at 0)
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-0",
prompt=SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
original_prompt={"prompt": "hello 0"},
sampling_params_list=[_sampling_params(), _sampling_params()],
final_stage_id=1,
)
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
assert len(stage0_r1.add_request_calls) == 0
# Request 1 → should land on replica 1 (RR advances)
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-1",
prompt=SimpleNamespace(request_id="req-1", prompt_token_ids=[5, 6]),
original_prompt={"prompt": "hello 1"},
sampling_params_list=[_sampling_params(), _sampling_params()],
final_stage_id=1,
)
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 1)
assert len(stage0_r0.add_request_calls) == 1 # unchanged
# Complete req-0 at stage-0 replica-0 → should forward to stage-1
stage0_r0.push_engine_core_outputs(_engine_core_outputs("s0r0-raw", 1.0))
await _wait_for(lambda: len(stage1.add_request_calls) == 1)
assert stage1.add_request_calls[0][0].request_id == "req-0"
# Complete req-0 at stage-1 → final output
proc1.request_outputs = [_build_request_output("req-0", token_ids=[10], finished=True)]
stage1.push_engine_core_outputs(_engine_core_outputs("s1-raw", 2.0))
output_msg = await _get_output_message(orchestrator_fixture)
assert output_msg["request_id"] == "req-0"
assert output_msg["stage_id"] == 1
assert output_msg["finished"] is True
assert "req-0" not in orchestrator_fixture.orchestrator.request_states
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_multi_replica_abort_broadcasts_to_all_replicas(orchestrator_factory) -> None:
"""Abort must be sent to every replica across all stages."""
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
stage1 = FakeStageClient(stage_type="llm", final_output=True)
proc0 = FakeOutputProcessor()
proc1 = FakeOutputProcessor()
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
stage_pools = _build_stage_pools(
[[stage0_r0, stage0_r1], [stage1]],
output_processors=[proc0, proc1],
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
)
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
try:
await _enqueue_add_request(
orchestrator_fixture,
request_id="req-abort-mr",
prompt=SimpleNamespace(request_id="req-abort-mr", prompt_token_ids=[1]),
original_prompt={"prompt": "cancel"},
sampling_params_list=[_sampling_params(), _sampling_params()],
final_stage_id=1,
)
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
await _enqueue_abort_request(orchestrator_fixture, ["req-abort-mr"])
await _wait_for(lambda: bool(stage0_r0.abort_calls))
assert stage0_r0.abort_calls == [["req-abort-mr"]]
assert stage0_r1.abort_calls == []
assert stage1.abort_calls == []
assert "req-abort-mr" not in orchestrator_fixture.orchestrator.request_states
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_multi_replica_shutdown_all_replicas(orchestrator_factory) -> None:
"""Shutdown must shut down every replica across all stages."""
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
stage1 = FakeStageClient(stage_type="llm", final_output=True)
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
stage_pools = _build_stage_pools(
[[stage0_r0, stage0_r1], [stage1]],
stage_vllm_configs=[default_vllm_cfg, default_vllm_cfg],
)
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
await _shutdown_orchestrator(orchestrator_fixture)
assert not orchestrator_fixture.thread.is_alive()
for client in [stage0_r0, stage0_r1, stage1]:
assert client.shutdown_calls == 1
@pytest.mark.asyncio
async def test_stage_pool_submit_update_reuses_existing_binding() -> None:
"""A request admitted to one replica must keep using that replica on updates."""
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
pool = StagePool(
0,
[stage0_r0, stage0_r1],
output_processor=FakeOutputProcessor(),
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
req0_state = OrchestratorRequestState(
request_id="req-0",
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
req1_state = OrchestratorRequestState(
request_id="req-1",
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await pool.submit_initial("req-0", req0_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]))
await pool.submit_update("req-0", req0_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[3]))
await pool.submit_initial("req-1", req1_state, SimpleNamespace(request_id="req-1", prompt_token_ids=[4, 5]))
await pool.submit_update("req-1", req1_state, SimpleNamespace(request_id="req-1", prompt_token_ids=[6]))
assert pool.get_bound_replica_id("req-0") == 0
assert pool.get_bound_replica_id("req-1") == 1
assert len(stage0_r0.add_request_calls) == 2
assert len(stage0_r1.add_request_calls) == 2
assert stage0_r0.add_request_calls[0][0].request_id == "req-0"
assert stage0_r0.add_request_calls[1][0].request_id == "req-0"
assert stage0_r1.add_request_calls[0][0].request_id == "req-1"
assert stage0_r1.add_request_calls[1][0].request_id == "req-1"
@pytest.mark.asyncio
async def test_stage_pool_submit_update_refreshes_output_processor_state() -> None:
output_processor = FakeOutputProcessor()
class AssertingStageClient(FakeStageClient):
async def add_request_async(self, *args, **kwargs) -> None:
if len(self.add_request_calls) == 1:
prompts = [call_kwargs["prompt"] for _, call_kwargs in output_processor.add_request_calls]
assert prompts == ["seg-1", "seg-2"]
await super().add_request_async(*args, **kwargs)
stage0 = AssertingStageClient(stage_type="llm", final_output=False)
pool = StagePool(
0,
[stage0],
output_processor=output_processor,
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
req_state = OrchestratorRequestState(
request_id="req-0",
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await pool.submit_initial(
"req-0",
req_state,
SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]),
prompt_text="seg-1",
)
await pool.submit_update(
"req-0",
req_state,
SimpleNamespace(request_id="req-0", prompt_token_ids=[3], resumable=True),
prompt_text="seg-2",
)
assert len(output_processor.add_request_calls) == 2
assert output_processor.add_request_calls[1][1]["prompt"] == "seg-2"
@pytest.mark.asyncio
async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> None:
class RecordingPool:
def __init__(self) -> None:
self.calls: list[tuple[str, Any]] = []
async def submit_update(self, request_id, req_state, request, *, prompt_text=None) -> int:
self.calls.append((request_id, prompt_text))
return 0
pool = RecordingPool()
orchestrator = object.__new__(Orchestrator)
orchestrator.request_states = {
"req-stream": OrchestratorRequestState(
request_id="req-stream",
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
}
orchestrator.stage_pools = [pool]
await orchestrator._handle_streaming_update(
{
"request_id": "req-stream",
"prompt": SimpleNamespace(request_id="req-stream", prompt_token_ids=[1], resumable=True),
"sampling_params_list": [_sampling_params()],
"output_prompt_text": "segment-2",
}
)
assert pool.calls == [("req-stream", "segment-2")]
assert orchestrator.request_states["req-stream"].streaming.enabled is True
@pytest.mark.asyncio
async def test_stage_pool_submit_initial_rolls_back_output_processor_when_client_submit_fails() -> None:
class FailingStageClient(FakeStageClient):
async def add_request_async(self, *args, **_kwargs) -> None:
raise RuntimeError("submit failed")
class TrackingOutputProcessor(FakeOutputProcessor):
def __init__(self) -> None:
super().__init__()
self.added_request_ids: list[str] = []
self.removed_request_ids: list[str] = []
def add_request(self, request, *_args, **_kwargs) -> None:
self.added_request_ids.append(request.request_id)
def remove_request(self, request_id: str) -> None:
self.removed_request_ids.append(request_id)
client = FailingStageClient(stage_type="llm", final_output=False)
output_processor = TrackingOutputProcessor()
pool = StagePool(
0,
[client],
output_processor=output_processor,
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
req_state = OrchestratorRequestState(
request_id="req-0",
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
with pytest.raises(RuntimeError, match="submit failed"):
await pool.submit_initial("req-0", req_state, SimpleNamespace(request_id="req-0", prompt_token_ids=[1, 2]))
assert output_processor.added_request_ids == ["req-0"]
assert output_processor.removed_request_ids == ["req-0"]
assert pool.get_bound_replica_id("req-0") is None
@pytest.mark.asyncio
async def test_stage_pool_abort_requests_logs_when_binding_is_missing(caplog) -> None:
stage0 = FakeStageClient(stage_type="llm", final_output=False)
pool = StagePool(
0,
[stage0],
output_processor=FakeOutputProcessor(),
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
target_logger = logging.getLogger("vllm_omni.engine.stage_pool")
target_logger.addHandler(caplog.handler)
prev_level = target_logger.level
target_logger.setLevel(logging.DEBUG)
try:
await pool.abort_requests(["missing-req"])
finally:
target_logger.removeHandler(caplog.handler)
target_logger.setLevel(prev_level)
assert not stage0.abort_calls
assert "abort: no binding for req=missing-req in stage-0" in caplog.text
@pytest.mark.asyncio
async def test_collective_rpc_ignores_invalid_stage_ids(orchestrator_factory, caplog) -> None:
stage0 = FakeCollectiveRpcStageClient(stage_type="llm", final_output=True, rpc_result={"stage": 0})
stage1 = FakeCollectiveRpcStageClient(stage_type="llm", final_output=True, rpc_result={"stage": 1})
stage_pools = _build_stage_pools(
[[stage0], [stage1]],
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
stage_vllm_configs=[
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
],
)
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
try:
target_logger = logging.getLogger("vllm_omni.engine.orchestrator")
target_logger.addHandler(caplog.handler)
prev_level = target_logger.level
target_logger.setLevel(logging.WARNING)
try:
orchestrator_fixture.request_sync_q.put_nowait(
{
"type": "collective_rpc",
"rpc_id": "rpc-1",
"method": "list_loras",
"stage_ids": [99, 1],
}
)
msg = await _get_rpc_message(orchestrator_fixture)
finally:
target_logger.removeHandler(caplog.handler)
target_logger.setLevel(prev_level)
assert msg["type"] == "collective_rpc_result"
assert msg["rpc_id"] == "rpc-1"
assert msg["stage_ids"] == [1]
assert msg["results"] == [{"stage": 1}]
assert not stage0.collective_rpc_calls
assert len(stage1.collective_rpc_calls) == 1
assert "collective_rpc: ignoring invalid stage_id 99" in caplog.text
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@pytest.mark.asyncio
async def test_multi_replica_cfg_companion_inherits_parent_affinity(orchestrator_factory) -> None:
"""CFG companions should be routed to the same stage-0 replica as their parent."""
stage0_r0 = FakeStageClient(stage_type="llm", final_output=False)
stage0_r1 = FakeStageClient(stage_type="llm", final_output=False)
default_vllm_cfg = SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))
stage_pools = _build_stage_pools(
[[stage0_r0, stage0_r1]],
output_processors=[FakeOutputProcessor()],
stage_vllm_configs=[default_vllm_cfg],
)
orchestrator_fixture = orchestrator_factory([], stage_pools=stage_pools)
try:
# Consume replica-0 first so the parent request binds to replica-1.
await _enqueue_add_request(
orchestrator_fixture,
request_id="warmup",
prompt=SimpleNamespace(request_id="warmup", prompt_token_ids=[0]),
original_prompt={"prompt": "warmup"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await _wait_for(lambda: len(stage0_r0.add_request_calls) == 1)
await _enqueue_add_request(
orchestrator_fixture,
request_id="parent",
prompt=SimpleNamespace(request_id="parent", prompt_token_ids=[1, 2]),
original_prompt={"prompt": "parent"},
sampling_params_list=[_sampling_params()],
final_stage_id=0,
)
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 1)
orchestrator_fixture.request_sync_q.put_nowait(
{
"type": "add_companion_request",
"companion_id": "parent-neg",
"parent_id": "parent",
"role": "negative",
"prompt": SimpleNamespace(request_id="parent-neg", prompt_token_ids=[9]),
"companion_prompt_text": {"prompt": "negative"},
"sampling_params_list": [_sampling_params()],
}
)
await _wait_for(lambda: len(stage0_r1.add_request_calls) == 2)
assert stage_pools[0].get_bound_replica_id("parent") == 1
assert stage_pools[0].get_bound_replica_id("parent-neg") == 1
assert len(stage0_r0.add_request_calls) == 1
assert stage0_r1.add_request_calls[0][0].request_id == "parent"
assert stage0_r1.add_request_calls[1][0].request_id == "parent-neg"
finally:
await _shutdown_orchestrator(orchestrator_fixture)
@@ -124,8 +124,8 @@ async def test_engine_dead_error_broadcasts_fatal_and_shuts_down(orchestrator_fa
@pytest.mark.asyncio
async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -> None:
"""When a diffusion stage returns an OmniRequestOutput with a non-None
error, the orchestrator must route it as a finished output message and
clean up the request state.
error, the orchestrator must route it as an error message and clean up
the request state.
"""
stage0 = FakeStageClient(stage_type="diffusion", final_output=True, final_output_type="image")
orchestrator_fixture = orchestrator_factory([stage0])
@@ -148,10 +148,10 @@ async def test_diffusion_error_output_routed_as_finished(orchestrator_factory) -
msg = await _get_any_output_message(orchestrator_fixture)
assert msg["type"] == "output"
assert msg["type"] == "error"
assert msg["request_id"] == "req-err"
assert msg["finished"] is True
assert msg["engine_outputs"].error == "gpu fault"
assert msg["stage_id"] == 0
assert msg["error"] == "gpu fault"
# Request state should be cleaned up.
await _wait_for(lambda: "req-err" not in orchestrator_fixture.orchestrator.request_states)
@@ -7,18 +7,18 @@ from vllm import SamplingParams
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _DummySenderStage:
stage_type = "llm"
final_output = False
def __init__(self, sender_info):
self._sender_info = sender_info
self.engine_outputs = None
def set_engine_outputs(self, outputs):
self.engine_outputs = outputs
def get_kv_sender_info(self):
return self._sender_info
@@ -26,6 +26,7 @@ class _DummySenderStage:
class _DummyDiffusionStage:
stage_type = "diffusion"
final_output = True
custom_process_input_func = None
def __init__(self, engine_input_source=None):
@@ -43,6 +44,15 @@ class _DummyDiffusionStage:
)
def _build_sender_pool(stage_id: int, sender_info: dict[str, object]) -> StagePool:
return StagePool(
stage_id,
_DummySenderStage(sender_info),
output_processor=object(),
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
)
def test_stage_engine_core_client_builds_kv_sender_info_from_tcp_address():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
@@ -126,14 +136,13 @@ def test_stage_engine_core_client_preserves_explicit_loopback_sender_host():
def test_forward_to_diffusion_attaches_kv_sender_info():
orchestrator = object.__new__(Orchestrator)
sender_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
diffusion_pool = StagePool(1, diffusion_stage)
orchestrator.num_stages = 2
orchestrator.stage_clients = [sender_stage, diffusion_stage]
orchestrator.stage_pools = [sender_pool, diffusion_pool]
orchestrator._cfg_tracker = CfgCompanionTracker()
orchestrator.stage_vllm_configs = [None, None]
orchestrator.output_processors = [None, None]
params = OmniDiffusionSamplingParams()
req_state = OrchestratorRequestState(
@@ -144,9 +153,8 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
)
output = SimpleNamespace(request_id="req-1", finished=True)
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-1", 0, output, req_state))
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-1", sender_pool.stage_id, output, req_state))
assert sender_stage.engine_outputs == [output]
assert diffusion_stage.calls[0]["request_id"] == "req-1"
assert diffusion_stage.calls[0]["kv_sender_info"] == {
0: {"host": "10.0.0.2", "zmq_port": 50151},
@@ -156,15 +164,14 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
orchestrator = object.__new__(Orchestrator)
source_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
previous_stage = _DummySenderStage({"host": "10.0.0.9", "zmq_port": 59999})
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
source_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
previous_pool = _build_sender_pool(1, {"host": "10.0.0.9", "zmq_port": 59999})
diffusion_pool = StagePool(2, diffusion_stage)
orchestrator.num_stages = 3
orchestrator.stage_clients = [source_stage, previous_stage, diffusion_stage]
orchestrator.stage_pools = [source_pool, previous_pool, diffusion_pool]
orchestrator._cfg_tracker = CfgCompanionTracker()
orchestrator.stage_vllm_configs = [None, None, None]
orchestrator.output_processors = [None, None, None]
params = OmniDiffusionSamplingParams()
req_state = OrchestratorRequestState(
@@ -175,9 +182,8 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
)
output = SimpleNamespace(request_id="req-3", finished=True)
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-3", 1, output, req_state))
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-3", previous_pool.stage_id, output, req_state))
assert previous_stage.engine_outputs == [output]
assert diffusion_stage.calls[0]["kv_sender_info"] == {
0: {"host": "10.0.0.2", "zmq_port": 50151},
}
@@ -185,9 +191,10 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
orchestrator = object.__new__(Orchestrator)
sender_stage = _DummySenderStage({"host": "10.0.0.2", "zmq_port": 50151})
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
diffusion_pool = StagePool(1, diffusion_stage)
class _AsyncQueue:
def __init__(self):
@@ -197,10 +204,8 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
self.items.append(item)
orchestrator.num_stages = 2
orchestrator.stage_clients = [sender_stage, diffusion_stage]
orchestrator.stage_pools = [sender_pool, diffusion_pool]
orchestrator._cfg_tracker = CfgCompanionTracker()
orchestrator.stage_vllm_configs = [None, None]
orchestrator.output_processors = [None, None]
orchestrator.output_async_queue = _AsyncQueue()
orchestrator.request_states = {}
orchestrator._pd_kv_params = {}
@@ -217,7 +222,6 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
output = SimpleNamespace(request_id="req-empty", finished=True)
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-empty", 0, output, req_state))
assert sender_stage.engine_outputs == [output]
assert diffusion_stage.calls == []
assert len(orchestrator.output_async_queue.items) == 1
terminal_msg = orchestrator.output_async_queue.items[0]
@@ -231,10 +235,11 @@ def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
def test_prewarm_diffusion_attaches_kv_sender_info():
orchestrator = object.__new__(Orchestrator)
sender_stage = _DummySenderStage({"host": "10.0.0.3", "zmq_port": 50151})
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
sender_pool = _build_sender_pool(0, {"host": "10.0.0.3", "zmq_port": 50151})
diffusion_pool = StagePool(1, diffusion_stage)
orchestrator.stage_clients = [sender_stage, diffusion_stage]
orchestrator.stage_pools = [sender_pool, diffusion_pool]
orchestrator.num_stages = 2
req_state = OrchestratorRequestState(
File diff suppressed because it is too large. Load diff
+50 -2
View File
@@ -38,6 +38,7 @@ def _make_headless_args() -> argparse.Namespace:
return argparse.Namespace(
model="fake-model",
stage_id=3,
replica_id=0,
omni_master_address="127.0.0.1",
omni_master_port=26000,
api_server_count=0,
@@ -45,6 +46,7 @@ def _make_headless_args() -> argparse.Namespace:
stage_configs_path=None,
log_stats=False,
disable_log_stats=False,
stage_init_timeout=600,
)
@@ -59,6 +61,7 @@ def test_run_headless_registers_stage_once_and_launches_all_local_engines(mocker
node_rank_within_dp=0,
)
vllm_config = mocker.Mock(parallel_config=parallel_config)
vllm_config.needs_dp_coordinator = False
executor_class = mocker.Mock()
engine_manager = mocker.Mock()
@@ -99,6 +102,7 @@ def test_run_headless_registers_stage_once_and_launches_all_local_engines(mocker
omni_stage_id=3,
omni_stage_config=stage_cfg,
coordinator=None,
replica_id=0,
)
mock_manager_cls.assert_called_once()
manager_kwargs = mock_manager_cls.call_args.kwargs
@@ -155,13 +159,56 @@ def test_run_headless_honors_explicit_log_stats_flag(mocker: MockerFixture) -> N
assert manager_kwargs["log_stats"] is True
def test_run_headless_registers_llm_replica_id(mocker: MockerFixture) -> None:
args = _make_headless_args()
args.replica_id = 2
stage_cfg = mocker.Mock(stage_id=3)
parallel_config = mocker.Mock(
data_parallel_size_local=1,
data_parallel_rank=0,
data_parallel_rank_local=0,
node_rank_within_dp=0,
)
vllm_config = mocker.Mock(parallel_config=parallel_config)
vllm_config.needs_dp_coordinator = False
engine_manager = mocker.Mock()
mocker.patch(
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
return_value=("/fake/stages.yaml", [stage_cfg]),
)
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
mocker.patch(
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
return_value=(None, None, None),
)
mocker.patch(
"vllm_omni.engine.stage_init_utils.build_vllm_config",
return_value=(vllm_config, mocker.Mock()),
)
mock_register = mocker.patch(
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
return_value="tcp://127.0.0.1:26001",
)
mocker.patch("vllm.v1.engine.utils.CoreEngineProcManager", return_value=engine_manager)
mocker.patch("signal.signal")
run_headless(args)
assert mock_register.call_args.kwargs["replica_id"] == 2
def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFixture) -> None:
args = _make_headless_args()
args.replica_id = 1
stage_cfg = mocker.Mock(stage_id=3, stage_type="diffusion")
stage_cfg.engine_args = mocker.Mock()
stage_cfg.engine_input_source = []
stage_cfgs = [stage_cfg]
metadata = mocker.Mock(stage_id=3)
metadata = mocker.Mock(stage_id=3, stage_type="diffusion")
od_config = mocker.Mock()
proc = mocker.Mock()
proc.exitcode = 0
@@ -199,6 +246,7 @@ def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFix
omni_stage_id=3,
omni_stage_config=stage_cfg,
return_addresses=True,
replica_id=1,
)
mock_spawn.assert_called_once_with(
"fake-model",
@@ -207,5 +255,5 @@ def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFix
request_address="tcp://127.0.0.1:26002",
response_address="tcp://127.0.0.1:26003",
)
mock_handshake.assert_called_once_with(proc, "tcp://127.0.0.1:26001")
mock_handshake.assert_called_once_with(proc, "tcp://127.0.0.1:26001", 600)
proc.join.assert_called_once_with()
+41
View File
@@ -360,6 +360,47 @@ class TestLoadAndResolveStageConfigs:
assert len(stage_configs) == 1
assert "dtype" in stage_configs[0]["engine_args"]
def test_stage_configs_path_promotes_new_deploy_yaml_without_expanding_replicas(
self, tmp_path, mocker: MockerFixture
):
deploy_path = tmp_path / "qwen3_multi.yaml"
deploy_path.write_text(
'stages:\n - stage_id: 0\n devices: "0"\n - stage_id: 1\n devices: "1,2,3"\n num_replicas: 3\n',
encoding="utf-8",
)
returned_stage_configs = [
create_config({"stage_id": 0, "runtime": {"devices": "0"}, "engine_args": {"model": "dummy"}}),
create_config(
{
"stage_id": 1,
"runtime": {"devices": "1,2,3", "num_replicas": 3},
"engine_args": {"model": "dummy"},
}
),
]
load_stage_configs = mocker.patch(
"vllm_omni.entrypoints.utils.load_stage_configs_from_model",
return_value=returned_stage_configs,
)
config_path, stage_configs = load_and_resolve_stage_configs(
model="dummy-model",
stage_configs_path=str(deploy_path),
kwargs={},
)
load_stage_configs.assert_called_once_with(
"dummy-model",
base_engine_args={},
deploy_config_path=str(deploy_path),
stage_overrides=None,
)
assert config_path == str(deploy_path)
assert len(stage_configs) == 2
assert stage_configs[1].runtime.num_replicas == 3
assert stage_configs[1].runtime.devices == "1,2,3"
class TestLoadStageConfigsFromYaml:
"""Regression tests for stage-config loading and merging."""
+64 -28
View File
@@ -385,7 +385,8 @@ class OmniServerStageCli(OmniServer):
self.stage_ids = stage_ids or self._load_stage_ids(resolved_cfg)
if 0 not in self.stage_ids:
raise ValueError(f"Stage CLI test requires stage_id=0 in config: {stage_config_path}")
self.stage_procs: dict[int, subprocess.Popen] = {}
self.stage_replica_counts = self._load_stage_replica_counts(resolved_cfg)
self.stage_procs: dict[tuple[int, int], subprocess.Popen] = {}
self.proc = None
@staticmethod
@@ -415,6 +416,19 @@ class OmniServerStageCli(OmniServer):
runtime_devices[int(stage_id)] = str(devices)
return runtime_devices
@staticmethod
def _load_stage_replica_counts(resolved_config: dict) -> dict[int, int]:
replica_counts: dict[int, int] = {}
for stage in OmniServerStageCli._stage_entries(resolved_config):
stage_id = stage.get("stage_id")
if stage_id is None:
continue
replica_counts[int(stage_id)] = max(
1,
int(stage.get("num_replicas") or stage.get("runtime", {}).get("num_replicas", 1)),
)
return replica_counts
@classmethod
def _parse_device_list(cls, devices: str | int) -> list[str]:
if isinstance(devices, int):
@@ -452,13 +466,29 @@ class OmniServerStageCli(OmniServer):
return ",".join(visible_device_list[idx] for idx in logical_ids)
def _set_stage_device_env(self, stage_id: int, env: dict[str, str], devices: str) -> None:
mapped_devices = self._map_stage_devices(stage_id, self.visible_device_list, devices)
def _devices_for_replica(self, stage_id: int, devices: str, replica_id: int) -> str:
replica_count = self.stage_replica_counts.get(stage_id, 1)
if replica_count == 1:
return devices
device_list = self._parse_device_list(devices)
if len(device_list) % replica_count != 0:
raise ValueError(
f"Stage {stage_id} has {len(device_list)} device(s) for {replica_count} replica(s); "
"device count must be divisible by replica count"
)
devices_per_replica = len(device_list) // replica_count
start = replica_id * devices_per_replica
return ",".join(device_list[start : start + devices_per_replica])
def _set_stage_device_env(self, stage_id: int, env: dict[str, str], devices: str, replica_id: int = 0) -> None:
replica_devices = self._devices_for_replica(stage_id, devices, replica_id)
mapped_devices = self._map_stage_devices(stage_id, self.visible_device_list, replica_devices)
env_var = getattr(current_omni_platform, "device_control_env_var", None)
if env_var:
env[env_var] = mapped_devices
def _build_stage_cmd(self, stage_id: int, *, headless: bool) -> list[str]:
def _build_stage_cmd(self, stage_id: int, *, headless: bool, replica_id: int = 0) -> list[str]:
cmd = [
sys.executable,
"-m",
@@ -474,6 +504,8 @@ class OmniServerStageCli(OmniServer):
self.host,
"--omni-master-port",
str(self.master_port),
"--replica-id",
str(replica_id),
]
if headless:
@@ -484,7 +516,7 @@ class OmniServerStageCli(OmniServer):
cmd += self.serve_args
return cmd
def _launch_stage(self, stage_id: int, *, headless: bool) -> None:
def _launch_stage(self, stage_id: int, *, headless: bool, replica_id: int = 0) -> None:
env = os.environ.copy()
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
if self.env_dict is not None:
@@ -492,19 +524,20 @@ class OmniServerStageCli(OmniServer):
devices = self.stage_runtime_devices.get(stage_id)
if devices:
self._set_stage_device_env(stage_id, env, devices)
self._set_stage_device_env(stage_id, env, devices, replica_id=replica_id)
cmd = self._build_stage_cmd(stage_id, headless=headless)
print(f"Launching OmniServerStageCli stage {stage_id}: {' '.join(cmd)}")
cmd = self._build_stage_cmd(stage_id, headless=headless, replica_id=replica_id)
print(f"Launching OmniServerStageCli stage {stage_id} replica {replica_id}: {' '.join(cmd)}")
# Capture each subprocess's stdout+stderr to a per-stage log file so
# debugging "Stage N exited before API server ready" doesn't rely on
# guessing; the file is surfaced in the RuntimeError message.
log_path = Path(tempfile.gettempdir()) / f"omni_stage_{stage_id}_{self.master_port}.log"
log_path = Path(tempfile.gettempdir()) / f"omni_stage_{stage_id}_replica_{replica_id}_{self.master_port}.log"
self._stage_log_paths = getattr(self, "_stage_log_paths", {})
self._stage_log_paths[stage_id] = log_path
stage_key = (stage_id, replica_id)
self._stage_log_paths[stage_key] = log_path
log_fh = open(log_path, "w", buffering=1) # noqa: SIM115 - closed in __exit__
self._stage_log_files = getattr(self, "_stage_log_files", {})
self._stage_log_files[stage_id] = log_fh
self._stage_log_files[stage_key] = log_fh
proc = subprocess.Popen(
cmd,
env=env,
@@ -512,36 +545,39 @@ class OmniServerStageCli(OmniServer):
stdout=log_fh,
stderr=subprocess.STDOUT,
)
self.stage_procs[stage_id] = proc
if stage_id == 0:
self.stage_procs[stage_key] = proc
if stage_id == 0 and replica_id == 0:
self.proc = proc
def _ensure_stage_processes_alive(self) -> None:
for stage_id, proc in self.stage_procs.items():
for (stage_id, replica_id), proc in self.stage_procs.items():
ret = proc.poll()
if ret is not None:
log_path = getattr(self, "_stage_log_paths", {}).get(stage_id)
log_path = getattr(self, "_stage_log_paths", {}).get((stage_id, replica_id))
tail = ""
if log_path and log_path.exists():
try:
with open(log_path, encoding="utf-8", errors="replace") as f:
lines = f.readlines()
tail = "\n=== Last 60 lines of stage {} log ({}) ===\n{}".format(
stage_id, log_path, "".join(lines[-60:]) or "<empty>"
tail = "\n=== Last 60 lines of stage {} replica {} log ({}) ===\n{}".format(
stage_id, replica_id, log_path, "".join(lines[-60:]) or "<empty>"
)
except Exception as exc: # pragma: no cover - diagnostic only
tail = f"\n<failed to read stage log {log_path}: {exc}>"
raise RuntimeError(f"Stage {stage_id} exited with code {ret} before API server became ready.{tail}")
raise RuntimeError(
f"Stage {stage_id} replica {replica_id} exited with code {ret} before API server became ready.{tail}"
)
def _start_server(self) -> None:
ordered_stage_ids = [0, *[stage_id for stage_id in self.stage_ids if stage_id != 0]]
self._launch_stage(0, headless=False)
self._launch_stage(0, headless=False, replica_id=0)
time.sleep(2)
self._ensure_stage_processes_alive()
for stage_id in ordered_stage_ids[1:]:
self._launch_stage(stage_id, headless=True)
for replica_id in range(self.stage_replica_counts.get(stage_id, 1)):
self._launch_stage(stage_id, headless=True, replica_id=replica_id)
max_wait = 1200
start_time = time.time()
@@ -568,15 +604,15 @@ class OmniServerStageCli(OmniServer):
whatever state the stage was in when it was torn down.
"""
log_paths = getattr(self, "_stage_log_paths", {}) or {}
for stage_id in sorted(log_paths):
log_path = log_paths[stage_id]
for stage_id, replica_id in sorted(log_paths):
log_path = log_paths[(stage_id, replica_id)]
if not log_path or not log_path.exists():
continue
try:
with open(log_path, encoding="utf-8", errors="replace") as f:
lines = f.readlines()
except Exception as exc: # pragma: no cover - diagnostic only
print(f"[OmniServerStageCli] stage {stage_id} log read failed: {exc}", flush=True)
print(f"[OmniServerStageCli] stage {stage_id} replica {replica_id} log read failed: {exc}", flush=True)
continue
total = len(lines)
if total <= head_lines + tail_lines:
@@ -587,18 +623,18 @@ class OmniServerStageCli(OmniServer):
head_chunk = lines[:head_lines]
tail_chunk = lines[-tail_lines:]
elided = total - head_lines - tail_lines
print(f"\n=== stage {stage_id} log HEAD ({log_path}) ===", flush=True)
print(f"\n=== stage {stage_id} replica {replica_id} log HEAD ({log_path}) ===", flush=True)
print("".join(head_chunk).rstrip("\n"), flush=True)
if tail_chunk:
print(f"\n... [{elided} lines elided] ...", flush=True)
print(f"\n=== stage {stage_id} log TAIL ({log_path}) ===", flush=True)
print(f"\n=== stage {stage_id} replica {replica_id} log TAIL ({log_path}) ===", flush=True)
print("".join(tail_chunk).rstrip("\n"), flush=True)
print(f"=== end stage {stage_id} log ===\n", flush=True)
print(f"=== end stage {stage_id} replica {replica_id} log ===\n", flush=True)
def __exit__(self, exc_type, exc_val, exc_tb):
self._dump_stage_logs_for_debug()
for stage_id in sorted(self.stage_procs, reverse=True):
proc = self.stage_procs[stage_id]
for stage_key in sorted(self.stage_procs, reverse=True):
proc = self.stage_procs[stage_key]
if proc.poll() is None:
self._kill_process_tree(proc.pid)
run_forced_gpu_cleanup_round()
+73
View File
@@ -387,6 +387,79 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
},
},
},
"qwen3_omni_moe_multi_replicas_4gpu": {
"base_config": "qwen3_omni_moe.yaml",
"async_chunk": True,
"stages": [
{
"stage_id": 0,
"devices": "0",
"gpu_memory_utilization": 0.85,
"max_num_seqs": 6,
"max_model_len": 32768,
"mm_processor_cache_gb": 0,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 150, "ignore_eos": False},
},
{
"stage_id": 1,
"devices": "1,2,3",
"num_replicas": 3,
"gpu_memory_utilization": 0.6,
"max_num_seqs": 2,
"max_model_len": 32768,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 1000},
},
{
"stage_id": 2,
"devices": "1,2,3",
"num_replicas": 3,
"gpu_memory_utilization": 0.1,
"max_num_seqs": 2,
"max_num_batched_tokens": 65536,
"load_format": "dummy",
"default_sampling_params": {"max_tokens": 2000},
},
],
},
"bagel_multi_replicas_4gpu": {
"base_config": "bagel.yaml",
"async_chunk": False,
"stages": [
{
"stage_id": 0,
"devices": "0",
"max_num_seqs": 6,
"max_num_batched_tokens": 16384,
"gpu_memory_utilization": 0.45,
"load_format": "dummy",
"default_sampling_params": {
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"max_tokens": 256,
"detokenize": False,
},
},
{
"stage_id": 1,
"devices": "1,2,3",
"num_replicas": 3,
"max_num_seqs": 1,
"enforce_eager": True,
"gpu_memory_utilization": 0.7,
"load_format": "dummy",
"default_sampling_params": {
"seed": 42,
"num_inference_steps": 2,
"guidance_scale": 0.0,
"height": 512,
"width": 512,
},
},
],
},
"bagel": {
"base_config": "bagel.yaml",
"stages": [
+56
View File
@@ -0,0 +1,56 @@
"""Tests for OmniServerStageCli process planning helpers."""
from __future__ import annotations
from pathlib import Path
from tests.helpers.runtime import OmniServerStageCli
def _write_stage_config(tmp_path: Path) -> str:
path = tmp_path / "stages.yaml"
path.write_text(
"""
stages:
- stage_id: 0
devices: "0"
- stage_id: 1
devices: "1,2,3"
num_replicas: 3
""".strip(),
encoding="utf-8",
)
return str(path)
def test_stage_cli_builds_headless_replica_cmd(tmp_path: Path) -> None:
server = OmniServerStageCli("fake-model", _write_stage_config(tmp_path), ["--disable-log-stats"])
cmd = server._build_stage_cmd(1, headless=True, replica_id=2)
assert "--headless" in cmd
assert cmd[cmd.index("--stage-id") + 1] == "1"
assert cmd[cmd.index("--replica-id") + 1] == "2"
assert cmd[cmd.index("--stage-configs-path") + 1] == server.stage_config_path
def test_stage_cli_splits_devices_per_replica(tmp_path: Path) -> None:
server = OmniServerStageCli("fake-model", _write_stage_config(tmp_path), [])
assert server._devices_for_replica(1, "1,2,3", 0) == "1"
assert server._devices_for_replica(1, "1,2,3", 1) == "2"
assert server._devices_for_replica(1, "1,2,3", 2) == "3"
def test_stage_cli_maps_visible_devices_after_replica_split(tmp_path: Path) -> None:
server = OmniServerStageCli(
"fake-model",
_write_stage_config(tmp_path),
[],
env_dict={"CUDA_VISIBLE_DEVICES": "4,5,6,7"},
)
env: dict[str, str] = {}
server._set_stage_device_env(1, env, "1,2,3", replica_id=1)
assert env["CUDA_VISIBLE_DEVICES"] == "6"
@@ -44,16 +44,12 @@ def _transfer_manager(
def test_text2flow_supports_batched_source_outputs():
stage_list = [
SimpleNamespace(
engine_outputs=[
_source_output("req-0", [10, 11], [1, 2, 3], {"speech_token": torch.tensor([[1, 2]])}),
_source_output("req-1", [20, 21], [4, 5], {"speech_token": torch.tensor([[3, 4]])}),
]
)
source_outputs = [
_source_output("req-0", [10, 11], [1, 2, 3], {"speech_token": torch.tensor([[1, 2]])}),
_source_output("req-1", [20, 21], [4, 5], {"speech_token": torch.tensor([[3, 4]])}),
]
outputs = text2flow(stage_list=stage_list, engine_input_source=[0], prompt=None)
outputs = text2flow(source_outputs=source_outputs, prompt=None)
assert len(outputs) == 2
assert outputs[0]["prompt_token_ids"] == [1, 2, 3]
@@ -32,11 +32,6 @@ def _source_output(token_ids: list[int], mm_output: dict | None = None):
)
def _stage_with_outputs(outputs):
"""Create a stage list entry with engine_outputs."""
return SimpleNamespace(engine_outputs=outputs)
# =============================================================================
# Tests for _has_source_image
# =============================================================================
@@ -285,11 +280,11 @@ class TestAr2Diffusion:
"""Test basic text-to-image pipeline: AR -> Diffusion."""
# 1024x1024 t2i: small(256) + large(1024) + EOS
token_ids = list(range(256)) + list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
source_outputs = [_source_output(token_ids)]
prompt = {"prompt": "a cat", "mm_processor_kwargs": {"target_h": 1024, "target_w": 1024}}
result = ar2diffusion(stage_list, [0], prompt=[prompt])
result = ar2diffusion(source_outputs, prompt=[prompt])
assert len(result) == 1
assert result[0]["prompt"] == "a cat"
assert result[0]["height"] == 1024
@@ -300,7 +295,7 @@ class TestAr2Diffusion:
"""Test image-to-image with prior_token_image_ids from AR model."""
token_ids = list(range(1024)) + [16385]
mm_output = {"ids": {"prior_image": torch.tensor([1, 2, 3])}}
stage_list = [_stage_with_outputs([_source_output(token_ids, mm_output)])]
source_outputs = [_source_output(token_ids, mm_output)]
from PIL import Image
@@ -311,14 +306,14 @@ class TestAr2Diffusion:
"multi_modal_data": {"image": img},
}
result = ar2diffusion(stage_list, [0], prompt=[prompt])
result = ar2diffusion(source_outputs, prompt=[prompt])
assert len(result) == 1
assert result[0]["extra"]["prior_token_image_ids"] is not None
def test_i2i_detected_via_modalities(self):
"""Test i2i mode detected via modalities field."""
token_ids = list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
source_outputs = [_source_output(token_ids)]
prompt = {
"prompt": "edit this",
@@ -326,35 +321,26 @@ class TestAr2Diffusion:
"modalities": ["img2img"],
}
result = ar2diffusion(stage_list, [0], prompt=[prompt])
result = ar2diffusion(source_outputs, prompt=[prompt])
assert len(result) == 1
def test_empty_engine_input_source_raises(self):
with pytest.raises(ValueError, match="engine_input_source cannot be empty"):
ar2diffusion([], [], prompt={})
def test_invalid_stage_id_raises(self):
with pytest.raises(IndexError, match="Invalid stage_id"):
ar2diffusion([_stage_with_outputs(None)], [5], prompt={})
def test_no_outputs_raises(self):
with pytest.raises(RuntimeError, match="has no outputs yet"):
ar2diffusion([SimpleNamespace(engine_outputs=None)], [0], prompt={})
def test_empty_source_outputs_returns_empty_list(self):
assert ar2diffusion([], prompt={}) == []
def test_default_dimensions(self):
"""When no height/width in prompt, defaults to 1024x1024."""
token_ids = list(range(256)) + list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
source_outputs = [_source_output(token_ids)]
prompt = {"prompt": "test"}
result = ar2diffusion(stage_list, [0], prompt=[prompt])
result = ar2diffusion(source_outputs, prompt=[prompt])
assert result[0]["height"] == 1024
assert result[0]["width"] == 1024
def test_requires_multimodal_data_with_pil_image(self):
"""Test that pil_image is included when requires_multimodal_data=True."""
token_ids = list(range(256)) + list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
source_outputs = [_source_output(token_ids)]
from PIL import Image
@@ -364,13 +350,13 @@ class TestAr2Diffusion:
"multi_modal_data": {"image": img},
}
result = ar2diffusion(stage_list, [0], prompt=[prompt], requires_multimodal_data=True)
result = ar2diffusion(source_outputs, prompt=[prompt], requires_multimodal_data=True)
assert result[0]["pil_image"] is img
def test_extra_params_passed_through(self):
"""Test that seed, num_inference_steps, guidance_scale, negative_prompt are passed."""
token_ids = list(range(256)) + list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(token_ids)])]
source_outputs = [_source_output(token_ids)]
prompt = {
"prompt": "test",
@@ -380,7 +366,7 @@ class TestAr2Diffusion:
"negative_prompt": "blurry",
}
result = ar2diffusion(stage_list, [0], prompt=[prompt])
result = ar2diffusion(source_outputs, prompt=[prompt])
assert result[0]["seed"] == 42
assert result[0]["num_inference_steps"] == 50
assert result[0]["guidance_scale"] == 7.5
@@ -390,14 +376,14 @@ class TestAr2Diffusion:
"""Test processing multiple requests in a batch."""
tokens1 = list(range(256)) + list(range(1024)) + [16385]
tokens2 = list(range(256)) + list(range(1024)) + [16385]
stage_list = [_stage_with_outputs([_source_output(tokens1), _source_output(tokens2)])]
source_outputs = [_source_output(tokens1), _source_output(tokens2)]
prompts = [
{"prompt": "first", "mm_processor_kwargs": {"target_h": 1024, "target_w": 1024}},
{"prompt": "second", "mm_processor_kwargs": {"target_h": 512, "target_w": 512}},
]
result = ar2diffusion(stage_list, [0], prompt=prompts)
result = ar2diffusion(source_outputs, prompt=prompts)
assert len(result) == 2
assert result[0]["prompt"] == "first"
assert result[1]["prompt"] == "second"
@@ -17,12 +17,11 @@ from vllm_omni.model_executor.stage_input_processors.mimo_audio import (
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_stage_list(codec_codes: torch.Tensor, request_id: str = "req-0"):
"""Build a minimal stage_list[0] with a single talker output carrying codec_codes."""
def _make_source_outputs(codec_codes: torch.Tensor, request_id: str = "req-0"):
output = SimpleNamespace(multimodal_output={"codes": {"audio": codec_codes}})
"""Build minimal source_outputs with one talker output carrying codec codes."""
talker_output = SimpleNamespace(outputs=[output], request_id=request_id)
stage0 = SimpleNamespace(engine_outputs=[talker_output])
return [stage0]
return [talker_output]
def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
@@ -33,7 +32,7 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
frames = (MAX_CODE2WAV_TOKENS // 36) + 100
codec_codes = torch.ones(frames, 1, 8, 4, dtype=torch.long)
stage_list = _make_stage_list(codec_codes, request_id="req-long")
source_outputs = _make_source_outputs(codec_codes, request_id="req-long")
# Attach caplog's handler directly to the module logger so the warning is
# captured regardless of propagation (vllm's logger configuration can
@@ -43,7 +42,7 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
prev_level = target_logger.level
target_logger.setLevel(logging.WARNING)
try:
prompts = llm2code2wav(stage_list, engine_input_source=[0])
prompts = llm2code2wav(source_outputs)
finally:
target_logger.removeHandler(caplog.handler)
target_logger.setLevel(prev_level)
@@ -58,9 +57,9 @@ def test_llm2code2wav_truncates_when_flat_exceeds_max(caplog):
def test_llm2code2wav_short_sequence_unchanged():
"""Short codec sequences are returned without truncation."""
codec_codes = torch.ones(4, 1, 8, 4, dtype=torch.long)
stage_list = _make_stage_list(codec_codes, request_id="req-short")
source_outputs = _make_source_outputs(codec_codes, request_id="req-short")
prompts = llm2code2wav(stage_list, engine_input_source=[0])
prompts = llm2code2wav(source_outputs)
assert len(prompts) == 1
# 4 frames + 1 pad row, flattened col-major → well below the cap
@@ -342,7 +342,7 @@ def test_non_async_processor_prepends_ref_code_and_sets_trim_context():
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
)
prompts = talker2code2wav(stage_list=[stage], engine_input_source=[0])
prompts = talker2code2wav(stage.engine_outputs)
assert len(prompts) == 1
prompt = prompts[0]
@@ -388,7 +388,7 @@ def test_non_async_processor_filters_out_of_range_codec_values():
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
)
prompts = talker2code2wav(stage_list=[stage], engine_input_source=[0])
prompts = talker2code2wav(stage.engine_outputs)
assert len(prompts) == 1
prompt = prompts[0]
@@ -38,7 +38,7 @@ class TestGenerator2Tokenizer:
audio = torch.tensor([10, 20, 30, 40])
stage = _make_stage([_make_output([audio])])
result = generator2tokenizer([stage], engine_input_source=[0])
result = generator2tokenizer(stage.engine_outputs)
assert len(result) == 1
assert result[0]["prompt_token_ids"] == [10, 20, 30, 40]
@@ -49,27 +49,15 @@ class TestGenerator2Tokenizer:
audio = torch.tensor([[1, 2, 3], [4, 5, 6]]) # (2, 3)
stage = _make_stage([_make_output([audio])])
result = generator2tokenizer([stage], engine_input_source=[0])
result = generator2tokenizer(stage.engine_outputs)
assert result[0]["prompt_token_ids"] == [1, 2, 3, 4, 5, 6]
def test_empty_engine_input_source_raises(self):
"""Empty engine_input_source raises ValueError."""
stage = _make_stage([_make_output([torch.tensor([1])])])
with pytest.raises(ValueError, match="cannot be empty"):
generator2tokenizer([stage], engine_input_source=[])
def test_invalid_stage_id_raises(self):
"""Stage id beyond stage_list length raises IndexError."""
stage = _make_stage([_make_output([torch.tensor([1])])])
with pytest.raises(IndexError, match="Invalid stage_id"):
generator2tokenizer([stage], engine_input_source=[5])
def test_no_outputs_yet_raises(self):
"""Stage with engine_outputs=None raises RuntimeError."""
stage = _make_stage(engine_outputs=None)
with pytest.raises(RuntimeError, match="no outputs yet"):
generator2tokenizer([stage], engine_input_source=[0])
with pytest.raises(TypeError):
generator2tokenizer(stage.engine_outputs)
# ---- Helpers for generator2tokenizer_async_chunk (streaming) ----
+21
View File
@@ -920,6 +920,27 @@ class TestDeployConfigLoading:
assert s0.yaml_engine_args["engine_output_type"] == "latent"
assert s0.yaml_extras["default_sampling_params"]["detokenize"] is True
def test_merge_pipeline_deploy_preserves_num_replicas(self, tmp_path):
from pathlib import Path
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
pipeline = _PIPELINE_REGISTRY["qwen3_omni_moe"]
base = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "qwen3_omni_moe.yaml"
if not base.exists():
pytest.skip("Deploy config not found")
overlay = tmp_path / "multi_replicas.yaml"
overlay.write_text(f'base_config: {base}\nstages:\n - stage_id: 1\n devices: "1,2"\n num_replicas: 2\n')
deploy = load_deploy_config(overlay)
assert deploy.stages[1].num_replicas == 2
stages = merge_pipeline_deploy(pipeline, deploy)
assert stages[1].yaml_runtime["devices"] == "1,2"
assert stages[1].yaml_runtime["num_replicas"] == 2
class TestQwen3OmniPipeline:
def test_registered(self):
+3 -3
View File
@@ -354,15 +354,15 @@ class TestFinishedLoadReqsDrain(unittest.TestCase):
class TestLoadCustomFuncSelection(unittest.TestCase):
def test_skips_legacy_stage_list_processors_for_full_payload_mode(self):
legacy_paths = [
def test_skips_non_payload_stage_input_processors_for_full_payload_mode(self):
incompatible_paths = [
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav",
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2dit",
"vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow",
"vllm_omni.model_executor.stage_input_processors.glm_image.ar2diffusion",
]
for func_path in legacy_paths:
for func_path in incompatible_paths:
selected_path, func = MixinHost._load_custom_func(
SimpleNamespace(
async_chunk=False,