[large-scale-serving] Integrate OmniCoordinator into stage engine pipeline (#3569)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: herotai214 <herotai214@gmail.com> Co-authored-by: herotai214 <herotai214@gmail.com>
This commit is contained in:
1 parent
ca9fd0b71c
commit
e277feacaf
33 files changed
+3033
-782
No files matched your search
@@ -136,6 +136,7 @@ def _make_engine_with_loop(
|
||||
calls and a real ``RequestScheduler``.
|
||||
"""
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine._closed = False
|
||||
engine.executor = _ConcurrencyTrackingExecutor(rpc_delay=rpc_delay)
|
||||
|
||||
sched = RequestScheduler()
|
||||
@@ -371,6 +372,7 @@ def test_collective_rpc_before_loop_starts_calls_executor_directly():
|
||||
on the caller's thread without enqueueing.
|
||||
"""
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine._closed = False
|
||||
engine._loop_started = False
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
@@ -391,6 +393,7 @@ def test_collective_rpc_before_loop_starts_serializes_concurrent_callers():
|
||||
serialize them so they cannot race on the shared executor MQ pair.
|
||||
"""
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine._closed = False
|
||||
engine._loop_started = False
|
||||
engine._rpc_lock = threading.RLock()
|
||||
engine._cv = threading.Condition(engine._rpc_lock)
|
||||
|
||||
@@ -6,18 +6,18 @@ from time import time
|
||||
import pytest
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
InstanceInfo,
|
||||
LeastQueueLengthBalancer,
|
||||
RandomBalancer,
|
||||
ReplicaInfo,
|
||||
ReplicaStatus,
|
||||
RoundRobinBalancer,
|
||||
StageStatus,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_load_balancer_select_returns_valid_index():
|
||||
"""Verify RandomBalancer.select() returns a valid index for instances."""
|
||||
"""Verify RandomBalancer.select() returns a valid index for replicas."""
|
||||
# Task structure mirrors async_omni; RandomBalancer ignores task contents.
|
||||
task: dict = {
|
||||
"request_id": "test",
|
||||
@@ -26,30 +26,30 @@ def test_load_balancer_select_returns_valid_index():
|
||||
}
|
||||
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
replicas = [
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=1,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10003",
|
||||
output_addr="tcp://host:10003-out",
|
||||
stage_id=1,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=2,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
@@ -58,38 +58,38 @@ def test_load_balancer_select_returns_valid_index():
|
||||
|
||||
balancer = RandomBalancer()
|
||||
|
||||
index = balancer.select(task, instances)
|
||||
index = balancer.select(task, replicas)
|
||||
|
||||
assert isinstance(index, int)
|
||||
assert 0 <= index < len(instances)
|
||||
assert 0 <= index < len(replicas)
|
||||
|
||||
|
||||
def test_round_robin_balancer_cycles_instances():
|
||||
def test_round_robin_balancer_cycles_replicas():
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
replicas = [
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=2,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=1,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10003",
|
||||
output_addr="tcp://host:10003-out",
|
||||
stage_id=1,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
@@ -97,35 +97,35 @@ def test_round_robin_balancer_cycles_instances():
|
||||
]
|
||||
|
||||
balancer = RoundRobinBalancer()
|
||||
results = [balancer.select({}, instances) for _ in range(5)]
|
||||
results = [balancer.select({}, replicas) for _ in range(5)]
|
||||
|
||||
# Default start_index=0 => 0,1,2,0,1
|
||||
assert results == [0, 1, 2, 0, 1]
|
||||
|
||||
|
||||
def test_round_robin_balancer_empty_instances_raises():
|
||||
with pytest.raises(ValueError, match="instances must not be empty"):
|
||||
def test_round_robin_balancer_empty_replicas_raises():
|
||||
with pytest.raises(ValueError, match="replicas must not be empty"):
|
||||
RoundRobinBalancer().select({}, [])
|
||||
|
||||
|
||||
def test_round_robin_balancer_after_large_index_and_shorter_list():
|
||||
"""Large start_index % len(instances) then counter wraps with shorter list."""
|
||||
"""Large start_index % len(replicas) then counter wraps with shorter list."""
|
||||
now = time()
|
||||
two = [
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
@@ -138,30 +138,30 @@ def test_round_robin_balancer_after_large_index_and_shorter_list():
|
||||
|
||||
def test_least_queue_length_balancer_picks_min_queue():
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
replicas = [
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=2,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10003",
|
||||
output_addr="tcp://host:10003-out",
|
||||
stage_id=1,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=5,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
@@ -169,41 +169,41 @@ def test_least_queue_length_balancer_picks_min_queue():
|
||||
]
|
||||
|
||||
balancer = LeastQueueLengthBalancer()
|
||||
index = balancer.select({}, instances)
|
||||
index = balancer.select({}, replicas)
|
||||
assert index == 1
|
||||
|
||||
|
||||
def test_least_queue_length_balancer_empty_instances_raises():
|
||||
with pytest.raises(ValueError, match="instances must not be empty"):
|
||||
def test_least_queue_length_balancer_empty_replicas_raises():
|
||||
with pytest.raises(ValueError, match="replicas must not be empty"):
|
||||
LeastQueueLengthBalancer().select({}, [])
|
||||
|
||||
|
||||
def test_least_queue_length_balancer_equal_queues_uses_choice(mocker):
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
replicas = [
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=3,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=3,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10003",
|
||||
output_addr="tcp://host:10003-out",
|
||||
stage_id=1,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=3,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
@@ -214,21 +214,21 @@ def test_least_queue_length_balancer_equal_queues_uses_choice(mocker):
|
||||
"vllm_omni.distributed.omni_coordinator.load_balancer.random.choice",
|
||||
return_value=2,
|
||||
)
|
||||
assert balancer.select({}, instances) == 2
|
||||
assert balancer.select({}, replicas) == 2
|
||||
|
||||
|
||||
def test_least_queue_length_balancer_negative_queue_raises():
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
replicas = [
|
||||
ReplicaInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
status=ReplicaStatus.UP,
|
||||
queue_length=-1,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
]
|
||||
with pytest.raises(ValueError, match="queue_length must be non-negative"):
|
||||
LeastQueueLengthBalancer().select({}, instances)
|
||||
LeastQueueLengthBalancer().select({}, replicas)
|
||||
@@ -8,8 +8,8 @@ import pytest
|
||||
import zmq
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
InstanceList,
|
||||
OmniCoordClientForHub,
|
||||
ReplicaList,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -32,8 +32,8 @@ def _wait_for_condition(cond, timeout: float = 2.0, interval: float = 0.01) -> b
|
||||
return False
|
||||
|
||||
|
||||
def test_hub_client_caches_instance_list_from_pub():
|
||||
"""Verify OmniCoordClientForHub receives instance list updates from OmniCoordinator and caches for get_instance_list()."""
|
||||
def test_hub_client_caches_replica_list_from_pub():
|
||||
"""Verify OmniCoordClientForHub receives replica list updates from OmniCoordinator and caches for get_replica_list()."""
|
||||
ctx, pub, endpoint = _bind_pub()
|
||||
|
||||
client = OmniCoordClientForHub(endpoint)
|
||||
@@ -41,7 +41,7 @@ def test_hub_client_caches_instance_list_from_pub():
|
||||
time.sleep(0.2)
|
||||
|
||||
now = time.time()
|
||||
instances_payload = [
|
||||
replicas_payload = [
|
||||
{
|
||||
"input_addr": "tcp://stage:10001",
|
||||
"output_addr": "tcp://stage:10001-out",
|
||||
@@ -71,37 +71,37 @@ def test_hub_client_caches_instance_list_from_pub():
|
||||
},
|
||||
]
|
||||
|
||||
payload = {"instances": instances_payload, "timestamp": now}
|
||||
payload = {"replicas": replicas_payload, "timestamp": now}
|
||||
pub.send(json.dumps(payload).encode("utf-8"))
|
||||
|
||||
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 3)
|
||||
assert _wait_for_condition(lambda: len(client.get_replica_list().replicas) == 3)
|
||||
|
||||
inst_list = client.get_instance_list()
|
||||
assert isinstance(inst_list, InstanceList)
|
||||
assert len(inst_list.instances) == 3
|
||||
rep_list = client.get_replica_list()
|
||||
assert isinstance(rep_list, ReplicaList)
|
||||
assert len(rep_list.replicas) == 3
|
||||
|
||||
for src, inst in zip(instances_payload, inst_list.instances, strict=True):
|
||||
assert inst.input_addr == src["input_addr"]
|
||||
assert inst.output_addr == src["output_addr"]
|
||||
assert inst.stage_id == src["stage_id"]
|
||||
assert inst.status.value == src["status"]
|
||||
for src, rep in zip(replicas_payload, rep_list.replicas, strict=True):
|
||||
assert rep.input_addr == src["input_addr"]
|
||||
assert rep.output_addr == src["output_addr"]
|
||||
assert rep.stage_id == src["stage_id"]
|
||||
assert rep.status.value == src["status"]
|
||||
|
||||
stage0 = client.get_instances_for_stage(0)
|
||||
stage1 = client.get_instances_for_stage(1)
|
||||
stage0 = client.get_replicas_for_stage(0)
|
||||
stage1 = client.get_replicas_for_stage(1)
|
||||
|
||||
assert all(inst.stage_id == 0 for inst in stage0.instances)
|
||||
assert all(inst.stage_id == 1 for inst in stage1.instances)
|
||||
assert all(rep.stage_id == 0 for rep in stage0.replicas)
|
||||
assert all(rep.stage_id == 1 for rep in stage1.replicas)
|
||||
|
||||
# Send an updated list with fewer instances and verify cache refresh.
|
||||
# Send an updated list with fewer replicas and verify cache refresh.
|
||||
updated_payload = {
|
||||
"instances": instances_payload[:2],
|
||||
"replicas": replicas_payload[:2],
|
||||
"timestamp": now + 1.0,
|
||||
}
|
||||
pub.send(json.dumps(updated_payload).encode("utf-8"))
|
||||
|
||||
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 2)
|
||||
updated_list = client.get_instance_list()
|
||||
assert len(updated_list.instances) == 2
|
||||
assert _wait_for_condition(lambda: len(client.get_replica_list().replicas) == 2)
|
||||
updated_list = client.get_replica_list()
|
||||
assert len(updated_list.replicas) == 2
|
||||
|
||||
client.close()
|
||||
pub.close(0)
|
||||
|
||||
@@ -9,7 +9,7 @@ import zmq
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
OmniCoordClientForStage,
|
||||
StageStatus,
|
||||
ReplicaStatus,
|
||||
)
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
omni_coord_client_for_stage as stage_client_module,
|
||||
@@ -47,7 +47,7 @@ def test_stage_client_auto_register_on_init():
|
||||
event = _recv_event(router)
|
||||
|
||||
assert event["event_type"] == "update"
|
||||
assert event["status"] == StageStatus.UP.value
|
||||
assert event["status"] == ReplicaStatus.UP.value
|
||||
assert event["stage_id"] == stage_id
|
||||
assert event["input_addr"] == input_addr
|
||||
assert event["output_addr"] == output_addr
|
||||
@@ -70,13 +70,13 @@ def test_stage_client_update_info_sends_correct_event():
|
||||
# Discard initial registration event.
|
||||
_recv_event(router)
|
||||
|
||||
client.update_info(status=StageStatus.ERROR)
|
||||
client.update_info(status=ReplicaStatus.ERROR)
|
||||
client.update_info(queue_length=10)
|
||||
|
||||
first = _recv_event(router)
|
||||
second = _recv_event(router)
|
||||
|
||||
assert first["status"] == StageStatus.ERROR.value
|
||||
assert first["status"] == ReplicaStatus.ERROR.value
|
||||
assert first["stage_id"] == stage_id
|
||||
assert first["input_addr"] == input_addr
|
||||
assert first["output_addr"] == output_addr
|
||||
@@ -107,7 +107,7 @@ def test_stage_client_close_sends_down_status():
|
||||
client.close()
|
||||
|
||||
event = _recv_event(router)
|
||||
assert event["status"] == StageStatus.DOWN.value
|
||||
assert event["status"] == ReplicaStatus.DOWN.value
|
||||
assert event["stage_id"] == stage_id
|
||||
assert event["input_addr"] == input_addr
|
||||
assert event["output_addr"] == output_addr
|
||||
@@ -274,6 +274,7 @@ def test_heartbeat_loop_retries_after_transient_send_failure():
|
||||
client._closed = False
|
||||
client._heartbeat_interval = 0.0
|
||||
client._stop_event = _FakeStopEvent()
|
||||
client._on_heartbeat = None
|
||||
|
||||
calls = {"count": 0}
|
||||
|
||||
|
||||
@@ -11,14 +11,14 @@ from vllm.v1.utils import get_engine_client_zmq_addr
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
OmniCoordClientForStage,
|
||||
OmniCoordinator,
|
||||
StageStatus,
|
||||
ReplicaStatus,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _recv_instance_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None:
|
||||
"""Receive InstanceList JSON from SUB socket. Returns None on timeout."""
|
||||
def _recv_replica_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None:
|
||||
"""Receive ReplicaList JSON from SUB socket. Returns None on timeout."""
|
||||
sub.setsockopt(zmq.RCVTIMEO, timeout_ms)
|
||||
try:
|
||||
data = sub.recv()
|
||||
@@ -27,16 +27,16 @@ def _recv_instance_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
def _wait_for_instance_list(
|
||||
def _wait_for_replica_list(
|
||||
sub: zmq.Socket,
|
||||
expected_count: int,
|
||||
timeout: float = 3.0,
|
||||
) -> dict | None:
|
||||
"""Wait until received InstanceList with expected_count active instances."""
|
||||
"""Wait until received ReplicaList with expected_count active replicas."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
msg = _recv_instance_list(sub, timeout_ms=500)
|
||||
if msg is not None and len(msg.get("instances", [])) == expected_count:
|
||||
msg = _recv_replica_list(sub, timeout_ms=500)
|
||||
if msg is not None and len(msg.get("replicas", [])) == expected_count:
|
||||
return msg
|
||||
return None
|
||||
|
||||
@@ -45,7 +45,7 @@ def _drain_sub_messages(sub: zmq.Socket, max_seconds: float = 0.4) -> None:
|
||||
"""Drain queued SUB messages for a short window."""
|
||||
deadline = time.time() + max_seconds
|
||||
while time.time() < deadline:
|
||||
_recv_instance_list(sub, timeout_ms=50)
|
||||
_recv_replica_list(sub, timeout_ms=50)
|
||||
|
||||
|
||||
def test_omni_coordinator_pub_coalescing_on_rapid_queue_updates():
|
||||
@@ -81,7 +81,7 @@ def test_omni_coordinator_pub_coalescing_on_rapid_queue_updates():
|
||||
)
|
||||
|
||||
# Wait for initial registration broadcast and clear any queued messages.
|
||||
msg = _wait_for_instance_list(sub, expected_count=1)
|
||||
msg = _wait_for_replica_list(sub, expected_count=1)
|
||||
assert msg is not None
|
||||
_drain_sub_messages(sub)
|
||||
|
||||
@@ -96,7 +96,7 @@ def test_omni_coordinator_pub_coalescing_on_rapid_queue_updates():
|
||||
deadline = time.time() + window_s
|
||||
recv_count = 0
|
||||
while time.time() < deadline:
|
||||
if _recv_instance_list(sub, timeout_ms=100) is not None:
|
||||
if _recv_replica_list(sub, timeout_ms=100) is not None:
|
||||
recv_count += 1
|
||||
|
||||
assert recv_count < update_count // 2, (
|
||||
@@ -110,8 +110,8 @@ def test_omni_coordinator_pub_coalescing_on_rapid_queue_updates():
|
||||
|
||||
|
||||
def test_omni_coordinator_registration_broadcast():
|
||||
"""Verify that after multiple OmniCoordClientForStage instances register,
|
||||
OmniCoordinator publishes an InstanceList containing all registered instances.
|
||||
"""Verify that after multiple OmniCoordClientForStage replicas register,
|
||||
OmniCoordinator publishes a ReplicaList containing all registered replicas.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
@@ -144,12 +144,12 @@ def test_omni_coordinator_registration_broadcast():
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10003", "tcp://stage:10003-out", 1),
|
||||
]
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected InstanceList with 3 instances"
|
||||
assert len(msg["instances"]) == 3
|
||||
msg = _wait_for_replica_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected ReplicaList with 3 replicas"
|
||||
assert len(msg["replicas"]) == 3
|
||||
assert isinstance(msg["timestamp"], (int, float))
|
||||
|
||||
input_addrs = {inst["input_addr"] for inst in msg["instances"]}
|
||||
input_addrs = {rep["input_addr"] for rep in msg["replicas"]}
|
||||
assert "tcp://stage:10001" in input_addrs
|
||||
assert "tcp://stage:10002" in input_addrs
|
||||
assert "tcp://stage:10003" in input_addrs
|
||||
@@ -162,7 +162,7 @@ def test_omni_coordinator_registration_broadcast():
|
||||
|
||||
|
||||
def test_omni_coordinator_heartbeat_timeout_handling():
|
||||
"""Verify that when a stage instance stops sending heartbeats,
|
||||
"""Verify that when a stage replica stops sending heartbeats,
|
||||
OmniCoordinator marks it as unhealthy and excludes it from the active list.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
@@ -201,23 +201,23 @@ def test_omni_coordinator_heartbeat_timeout_handling():
|
||||
"output_addr": "tcp://stage:c-out",
|
||||
"stage_id": 0,
|
||||
"event_type": "update",
|
||||
"status": StageStatus.UP.value,
|
||||
"status": ReplicaStatus.UP.value,
|
||||
"queue_length": 0,
|
||||
}
|
||||
dealer_c.send(json.dumps(reg_event).encode("utf-8"))
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected initial 3 instances"
|
||||
assert len(msg["instances"]) == 3
|
||||
msg = _wait_for_replica_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected initial 3 replicas"
|
||||
assert len(msg["replicas"]) == 3
|
||||
|
||||
# Wait for heartbeat timeout (timeout=5s, check interval ~2.5s).
|
||||
time.sleep(8.0)
|
||||
|
||||
# Receive the update (C should be ERROR and excluded from active list).
|
||||
msg_after_timeout = _wait_for_instance_list(sub, expected_count=2, timeout=5.0)
|
||||
assert msg_after_timeout is not None, "Expected InstanceList with 2 instances after timeout"
|
||||
instances = msg_after_timeout.get("instances", [])
|
||||
input_addrs = {inst["input_addr"] for inst in instances}
|
||||
msg_after_timeout = _wait_for_replica_list(sub, expected_count=2, timeout=5.0)
|
||||
assert msg_after_timeout is not None, "Expected ReplicaList with 2 replicas after timeout"
|
||||
replicas = msg_after_timeout.get("replicas", [])
|
||||
input_addrs = {rep["input_addr"] for rep in replicas}
|
||||
|
||||
assert "tcp://stage:a" in input_addrs
|
||||
assert "tcp://stage:b" in input_addrs
|
||||
@@ -232,8 +232,8 @@ def test_omni_coordinator_heartbeat_timeout_handling():
|
||||
sub_ctx.term()
|
||||
|
||||
|
||||
def test_omni_coordinator_instance_shutdown_handling():
|
||||
"""Verify that when a stage instance sends status='down',
|
||||
def test_omni_coordinator_replica_shutdown_handling():
|
||||
"""Verify that when a stage replica sends status='down',
|
||||
OmniCoordinator removes it from the active list and broadcasts an updated list.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
@@ -261,18 +261,18 @@ def test_omni_coordinator_instance_shutdown_handling():
|
||||
|
||||
client = OmniCoordClientForStage(router_addr, "tcp://stage:shutdown", "tcp://stage:shutdown-out", 0)
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=1)
|
||||
msg = _wait_for_replica_list(sub, expected_count=1)
|
||||
assert msg is not None
|
||||
assert len(msg["instances"]) == 1
|
||||
assert msg["instances"][0]["input_addr"] == "tcp://stage:shutdown"
|
||||
assert len(msg["replicas"]) == 1
|
||||
assert msg["replicas"][0]["input_addr"] == "tcp://stage:shutdown"
|
||||
|
||||
# Send down status (simulating graceful shutdown).
|
||||
client.update_info(status=StageStatus.DOWN)
|
||||
client.update_info(status=ReplicaStatus.DOWN)
|
||||
|
||||
# Receive updated list (should have 0 active instances).
|
||||
msg = _wait_for_instance_list(sub, expected_count=0)
|
||||
# Receive updated list (should have 0 active replicas).
|
||||
msg = _wait_for_replica_list(sub, expected_count=0)
|
||||
assert msg is not None
|
||||
assert len(msg["instances"]) == 0
|
||||
assert len(msg["replicas"]) == 0
|
||||
|
||||
client.close()
|
||||
coordinator.close()
|
||||
|
||||
@@ -171,18 +171,6 @@ def test_collect_initialized_clients_for_cleanup_deduplicates_clients():
|
||||
assert cleanup_clients == [shared, extra]
|
||||
|
||||
|
||||
def test_initialize_stages_rejects_non_diffusion_replicas_in_single_stage_mode():
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.single_stage_mode = True
|
||||
engine.stage_configs = [types.SimpleNamespace(stage_id=0, runtime={"num_replicas": 2})]
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="single_stage_mode only supports num_replicas > 1 for diffusion stages",
|
||||
):
|
||||
engine._validate_single_stage_mode_replica_constraints()
|
||||
|
||||
|
||||
def test_initialize_diffusion_replica_restores_device_visibility_after_local_init(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -427,6 +415,7 @@ def test_initialize_stages_cleans_up_successful_replicas_after_partial_multi_rep
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_server = None
|
||||
engine._coordinator_runtime = None
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
@@ -471,6 +460,7 @@ def test_initialize_stages_cleans_up_late_successful_replicas_after_early_multi_
|
||||
engine.single_stage_mode = False
|
||||
engine._single_stage_id_filter = None
|
||||
engine._omni_master_server = None
|
||||
engine._coordinator_runtime = None
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
|
||||
cfg0 = types.SimpleNamespace(model_config=types.SimpleNamespace(max_model_len=64))
|
||||
|
||||
@@ -1052,7 +1052,7 @@ async def test_stage_pool_abort_requests_logs_when_binding_is_missing(caplog) ->
|
||||
target_logger.setLevel(prev_level)
|
||||
|
||||
assert not stage0.abort_calls
|
||||
assert "abort: no binding for req=missing-req in stage-0" in caplog.text
|
||||
assert "abort: no live binding for req=missing-req in stage-0" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -203,6 +203,32 @@ class TestOmniMasterServerAllocation:
|
||||
server = OmniMasterServer(master_address="127.0.0.1", master_port=15004, stage_ids=[3])
|
||||
assert server.get_allocation(3) is server._stage_routes[(3, 0)]
|
||||
|
||||
def test_next_free_replica_id_skips_head_local_slot_until_filled(self):
|
||||
# Head pre-allocates slot (0, 0) for its own register_stage_with_omni_master
|
||||
# call. A same-host headless that registers with auto-assign BEFORE the
|
||||
# head's own registration must NOT be handed slot 0 — it should land on
|
||||
# slot 1 instead. Without the head_local_replicas reservation,
|
||||
# _next_free_replica_id would see (0, 0) absent from _stage_configs and
|
||||
# return 0, colliding with the head's own bound sockets.
|
||||
server = OmniMasterServer(
|
||||
master_address="127.0.0.1",
|
||||
master_port=15010,
|
||||
stage_ids=[0],
|
||||
head_local_replicas={0: [0]},
|
||||
)
|
||||
assert server._next_free_replica_id(0) == 1
|
||||
|
||||
def test_next_free_replica_id_uses_remote_slot_when_unowned(self):
|
||||
# When the head pre-allocates a remote-only slot (the head's _initialize_*
|
||||
# path waits on get_stage_config), auto-assign SHOULD fill it so the
|
||||
# head's wait unblocks. This is the original behavior, preserved.
|
||||
server = OmniMasterServer(
|
||||
master_address="127.0.0.1",
|
||||
master_port=15011,
|
||||
stage_ids=[0],
|
||||
)
|
||||
assert server._next_free_replica_id(0) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OmniMasterServer registration flow
|
||||
@@ -233,36 +259,6 @@ class TestOmniMasterServerRegistration:
|
||||
ctx.term()
|
||||
server.stop()
|
||||
|
||||
def test_server_handles_unknown_stage_id_gracefully(self):
|
||||
import msgspec
|
||||
import zmq
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
master_port = get_open_port()
|
||||
server = OmniMasterServer(master_address="127.0.0.1", master_port=master_port, stage_ids=[0])
|
||||
server.start()
|
||||
|
||||
ctx = zmq.Context()
|
||||
bad_sock = None
|
||||
good_sock = None
|
||||
try:
|
||||
bad_sock = ctx.socket(zmq.DEALER)
|
||||
bad_sock.connect(f"tcp://127.0.0.1:{master_port}")
|
||||
bad_sock.send(msgspec.msgpack.encode({"stage_id": 99}))
|
||||
assert not bad_sock.poll(timeout=500)
|
||||
|
||||
good_sock = ctx.socket(zmq.DEALER)
|
||||
good_sock.connect(f"tcp://127.0.0.1:{master_port}")
|
||||
good_sock.send(msgspec.msgpack.encode({"stage_id": 0}))
|
||||
assert good_sock.poll(timeout=2_000)
|
||||
good_sock.recv()
|
||||
finally:
|
||||
for sock in (bad_sock, good_sock):
|
||||
if sock is not None:
|
||||
sock.close(linger=0)
|
||||
ctx.term()
|
||||
server.stop()
|
||||
|
||||
def test_registration_stores_stage_config(self):
|
||||
import msgspec
|
||||
import zmq
|
||||
@@ -474,6 +470,8 @@ class TestSingleStageInitialization:
|
||||
engine._omni_master_address = "127.0.0.1"
|
||||
engine._omni_master_port = 26000
|
||||
engine._omni_master_server = None
|
||||
engine._omni_heartbeat_timeout = 30.0
|
||||
engine._coordinator_runtime = None
|
||||
engine.async_chunk = False
|
||||
engine.diffusion_batch_size = 2
|
||||
return engine
|
||||
@@ -508,10 +506,16 @@ class TestSingleStageInitialization:
|
||||
|
||||
def test_start_omni_master_server_uses_configured_stage_ids(self, mocker: MockerFixture):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
from vllm_omni.distributed import omni_coordinator as omni_coord_mod
|
||||
|
||||
engine = self._build_engine([], single_stage_mode=True, stage_id_filter=7)
|
||||
mock_oms = mocker.Mock(spec=OmniMasterServer)
|
||||
mocker.patch.object(engine_mod, "OmniMasterServer", return_value=mock_oms)
|
||||
mocker.patch.object(
|
||||
omni_coord_mod,
|
||||
"OmniCoordinatorRuntime",
|
||||
return_value=mocker.Mock(router_address="tcp://127.0.0.1:9999"),
|
||||
)
|
||||
|
||||
stage_plans = [
|
||||
_make_llm_plan(0, configured_stage_id=7, launch_mode="local"),
|
||||
@@ -520,12 +524,16 @@ class TestSingleStageInitialization:
|
||||
|
||||
engine._start_omni_master_server(stage_plans)
|
||||
|
||||
engine_mod.OmniMasterServer.assert_called_once_with(
|
||||
master_address="127.0.0.1",
|
||||
master_port=26000,
|
||||
stage_ids=[7, 11],
|
||||
stage_replica_counts={7: 1, 11: 1},
|
||||
)
|
||||
call_kwargs = engine_mod.OmniMasterServer.call_args.kwargs
|
||||
assert call_kwargs["master_address"] == "127.0.0.1"
|
||||
assert call_kwargs["master_port"] == 26000
|
||||
assert call_kwargs["stage_ids"] == [7, 11]
|
||||
assert call_kwargs["stage_replica_counts"] == {7: 1, 11: 1}
|
||||
# head_local_replicas reserves slots that the head will register
|
||||
# itself (launch_mode == "local"). Stage 11 is remote, so it must
|
||||
# NOT appear in the head-owned set — that slot is for the headless
|
||||
# to fill via auto-assign.
|
||||
assert call_kwargs["head_local_replicas"] == {7: [0]}
|
||||
mock_oms.start.assert_called_once()
|
||||
|
||||
def test_start_omni_master_server_duplicate_stage_ids_raise(self):
|
||||
@@ -545,7 +553,9 @@ class TestSingleStageInitialization:
|
||||
with pytest.raises(ValueError, match="requires both"):
|
||||
engine._start_omni_master_server([_make_llm_plan(0, configured_stage_id=7, launch_mode="local")])
|
||||
|
||||
def test_build_logical_stage_init_plans_clears_runtime_cfg_in_single_stage_mode(self, mocker: MockerFixture):
|
||||
def test_build_logical_stage_init_plans_preserves_runtime_cfg_for_local_llm_in_single_stage_mode(
|
||||
self, mocker: MockerFixture
|
||||
):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
|
||||
engine = self._build_engine([_make_stage_cfg(7)], single_stage_mode=True, stage_id_filter=7)
|
||||
@@ -574,7 +584,7 @@ class TestSingleStageInitialization:
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
|
||||
assert stage_plans[0].replicas[0].metadata.runtime_cfg is None
|
||||
assert stage_plans[0].replicas[0].metadata.runtime_cfg == {"devices": "0"}
|
||||
|
||||
def test_validate_single_stage_mode_allows_diffusion_replicas(self):
|
||||
stage_cfg = _make_stage_cfg(0, stage_type="diffusion")
|
||||
@@ -583,14 +593,6 @@ class TestSingleStageInitialization:
|
||||
|
||||
engine._validate_single_stage_mode_replica_constraints()
|
||||
|
||||
def test_validate_single_stage_mode_rejects_llm_replicas(self):
|
||||
stage_cfg = _make_stage_cfg(0, stage_type="llm")
|
||||
stage_cfg.runtime.num_replicas = 2
|
||||
engine = self._build_engine([stage_cfg], single_stage_mode=True, stage_id_filter=0)
|
||||
|
||||
with pytest.raises(ValueError, match="only supports num_replicas > 1 for diffusion"):
|
||||
engine._validate_single_stage_mode_replica_constraints()
|
||||
|
||||
def test_build_logical_stage_init_plans_preserves_diffusion_runtime_cfg_in_single_stage_mode(
|
||||
self, mocker: MockerFixture
|
||||
):
|
||||
@@ -809,6 +811,7 @@ class TestSingleStageReplicaInitialization:
|
||||
engine.model = "fake-model"
|
||||
engine.single_stage_mode = True
|
||||
engine._omni_master_server = mocker.Mock(spec=OmniMasterServer)
|
||||
engine._coordinator_runtime = None
|
||||
engine.stage_configs = []
|
||||
|
||||
fake_vllm_config = SimpleNamespace(parallel_config=SimpleNamespace())
|
||||
@@ -896,6 +899,7 @@ class TestSingleStageReplicaInitialization:
|
||||
engine._omni_master_server = mocker.Mock(spec=OmniMasterServer)
|
||||
engine._omni_master_server.address = "127.0.0.1"
|
||||
engine._omni_master_server.port = 25000
|
||||
engine._coordinator_runtime = None
|
||||
|
||||
plan = _make_diffusion_plan(0, configured_stage_id=5, launch_mode="local").replicas[0]
|
||||
sentinel_client = SimpleNamespace()
|
||||
@@ -950,6 +954,9 @@ class TestSingleStageReplicaInitialization:
|
||||
handshake_address="tcp://hs",
|
||||
request_address="tcp://req",
|
||||
response_address="tcp://resp",
|
||||
omni_coordinator_address=None,
|
||||
omni_stage_id=5,
|
||||
omni_replica_id=0,
|
||||
)
|
||||
mock_handshake.assert_called_once_with(proc, "tcp://hs", 60)
|
||||
mock_from_addresses.assert_called_once_with(
|
||||
@@ -972,6 +979,7 @@ class TestSingleStageReplicaInitialization:
|
||||
engine._omni_master_server = mocker.Mock(spec=OmniMasterServer)
|
||||
engine._omni_master_server.address = "127.0.0.1"
|
||||
engine._omni_master_server.port = 25000
|
||||
engine._coordinator_runtime = None
|
||||
|
||||
plan = _make_diffusion_plan(0, configured_stage_id=5, launch_mode="local").replicas[0]
|
||||
proc = mocker.Mock()
|
||||
|
||||
+294
-152
@@ -3,6 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -31,143 +33,118 @@ def test_serve_parser_accepts_no_async_chunk() -> None:
|
||||
assert args.async_chunk is False
|
||||
|
||||
|
||||
def _make_headless_args() -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_headless validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_headless_args(**overrides: Any) -> argparse.Namespace:
|
||||
"""Build an argparse.Namespace shaped like the headless CLI passes in.
|
||||
|
||||
Defaults pass every validation gate so individual tests can mutate just
|
||||
the field they're exercising.
|
||||
"""
|
||||
defaults = dict(
|
||||
model="fake-model",
|
||||
stage_id=3,
|
||||
stage_id=0,
|
||||
replica_id=0,
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
omni_replica_address=None,
|
||||
omni_dp_size_local=1,
|
||||
api_server_count=0,
|
||||
worker_backend="multi_process",
|
||||
stage_configs_path=None,
|
||||
deploy_config=None,
|
||||
log_stats=False,
|
||||
disable_log_stats=False,
|
||||
stage_init_timeout=600,
|
||||
tokenizer=None,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return argparse.Namespace(**defaults)
|
||||
|
||||
|
||||
def test_run_headless_registers_stage_once_and_launches_all_local_engines(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
stage_cfg = mocker.Mock(stage_id=3)
|
||||
stage_cfgs = [stage_cfg]
|
||||
parallel_config = mocker.Mock(
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=4,
|
||||
data_parallel_rank_local=1,
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = mocker.Mock(parallel_config=parallel_config)
|
||||
vllm_config.needs_dp_coordinator = False
|
||||
executor_class = mocker.Mock()
|
||||
engine_manager = mocker.Mock()
|
||||
def test_run_headless_requires_stage_id() -> None:
|
||||
args = _make_headless_args(stage_id=None)
|
||||
with pytest.raises(ValueError, match="--stage-id is required"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_requires_master_address() -> None:
|
||||
args = _make_headless_args(omni_master_address=None)
|
||||
with pytest.raises(ValueError, match="--omni-master-address and --omni-master-port"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_requires_master_port() -> None:
|
||||
args = _make_headless_args(omni_master_port=None)
|
||||
with pytest.raises(ValueError, match="--omni-master-address and --omni-master-port"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_rejects_multi_api_server_count() -> None:
|
||||
args = _make_headless_args(api_server_count=2)
|
||||
with pytest.raises(ValueError, match="api_server_count can't be set"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_rejects_non_multiprocess_worker_backend() -> None:
|
||||
args = _make_headless_args(worker_backend="ray")
|
||||
with pytest.raises(ValueError, match="worker_backend=multi_process"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_raises_when_stage_id_not_in_configs(mocker: MockerFixture) -> None:
|
||||
"""Headless looks up its assigned stage_id in the loaded deploy YAML and
|
||||
fails fast when the launcher's --stage-id doesn't match any entry."""
|
||||
other_stage = SimpleNamespace(stage_id=99)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", stage_cfgs),
|
||||
return_value=("/fake/stages.yaml", [other_stage]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mock_build_vllm_config = mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
return_value=(vllm_config, executor_class),
|
||||
)
|
||||
mock_register = mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value="tcp://127.0.0.1:26001",
|
||||
)
|
||||
mock_manager_cls = mocker.patch("vllm.v1.engine.utils.CoreEngineProcManager", return_value=engine_manager)
|
||||
mocker.patch("signal.signal")
|
||||
run_headless(args)
|
||||
|
||||
mock_build_vllm_config.assert_called_once_with(
|
||||
stage_cfg,
|
||||
"fake-model",
|
||||
stage_connector_spec={},
|
||||
engine_args_dict={},
|
||||
headless=True,
|
||||
)
|
||||
mock_register.assert_called_once_with(
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
omni_stage_id=3,
|
||||
omni_stage_config=stage_cfg,
|
||||
coordinator=None,
|
||||
replica_id=0,
|
||||
)
|
||||
mock_manager_cls.assert_called_once()
|
||||
manager_kwargs = mock_manager_cls.call_args.kwargs
|
||||
assert manager_kwargs["local_engine_count"] == 2
|
||||
assert manager_kwargs["start_index"] == 4
|
||||
assert manager_kwargs["local_start_index"] == 0
|
||||
assert manager_kwargs["local_client"] is False
|
||||
assert manager_kwargs["handshake_address"] == "tcp://127.0.0.1:26001"
|
||||
assert manager_kwargs["log_stats"] is False
|
||||
engine_manager.monitor_engine_liveness.assert_called_once_with()
|
||||
engine_manager.shutdown.assert_called_once_with()
|
||||
args = _make_headless_args(stage_id=0)
|
||||
with pytest.raises(ValueError, match="No stage config found for stage_id=0"):
|
||||
run_headless(args)
|
||||
|
||||
|
||||
def test_run_headless_honors_explicit_log_stats_flag(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
args.log_stats = True
|
||||
stage_cfg = mocker.Mock(stage_id=3)
|
||||
stage_cfgs = [stage_cfg]
|
||||
parallel_config = mocker.Mock(
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=4,
|
||||
data_parallel_rank_local=1,
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = mocker.Mock(parallel_config=parallel_config)
|
||||
executor_class = mocker.Mock()
|
||||
engine_manager = mocker.Mock()
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", stage_cfgs),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
return_value=(vllm_config, executor_class),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value="tcp://127.0.0.1:26001",
|
||||
)
|
||||
mock_manager_cls = mocker.patch("vllm.v1.engine.utils.CoreEngineProcManager", return_value=engine_manager)
|
||||
mocker.patch("signal.signal")
|
||||
run_headless(args)
|
||||
|
||||
manager_kwargs = mock_manager_cls.call_args.kwargs
|
||||
assert manager_kwargs["log_stats"] is True
|
||||
# ---------------------------------------------------------------------------
|
||||
# run_headless happy paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_run_headless_registers_llm_replica_id(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
args.replica_id = 2
|
||||
stage_cfg = mocker.Mock(stage_id=3)
|
||||
parallel_config = mocker.Mock(
|
||||
def _make_stage_cfg(stage_id: int, stage_type: str) -> SimpleNamespace:
|
||||
"""Build a stage config that satisfies every attribute run_headless reads.
|
||||
|
||||
Notably ``engine_args`` is a real dict (not a Mock) so
|
||||
``get_stage_devices_per_replica`` can call ``.get("tensor_parallel_size")``
|
||||
and feed the result through ``int()`` without TypeError.
|
||||
"""
|
||||
return SimpleNamespace(
|
||||
stage_id=stage_id,
|
||||
stage_type=stage_type,
|
||||
# No "devices" key -> split_devices_for_replicas skipped, each replica
|
||||
# inherits the launcher's CUDA_VISIBLE_DEVICES.
|
||||
runtime=None,
|
||||
engine_args={},
|
||||
)
|
||||
|
||||
|
||||
def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: MockerFixture) -> None:
|
||||
"""LLM headless: each loop iteration registers with auto-assigned
|
||||
replica_id (master picks a free slot) and spawns one
|
||||
``OmniCoreEngineProcManager`` per local replica."""
|
||||
from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse
|
||||
|
||||
stage_cfg = _make_stage_cfg(0, stage_type="llm")
|
||||
parallel_config = SimpleNamespace(
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
data_parallel_rank_local=0,
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = mocker.Mock(parallel_config=parallel_config)
|
||||
vllm_config.needs_dp_coordinator = False
|
||||
vllm_config = SimpleNamespace(parallel_config=parallel_config, needs_dp_coordinator=False)
|
||||
engine_manager = mocker.Mock()
|
||||
|
||||
mocker.patch(
|
||||
@@ -175,82 +152,247 @@ def test_run_headless_registers_llm_replica_id(mocker: MockerFixture) -> None:
|
||||
return_value=("/fake/stages.yaml", [stage_cfg]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
return_value=(vllm_config, mocker.Mock()),
|
||||
return_value=(vllm_config, object),
|
||||
)
|
||||
mock_register = mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value="tcp://127.0.0.1:26001",
|
||||
return_value=StageRegistrationResponse(
|
||||
handshake_address="tcp://127.0.0.1:26001",
|
||||
input_address="tcp://127.0.0.1:26002",
|
||||
output_address="tcp://127.0.0.1:26003",
|
||||
replica_id=0,
|
||||
coordinator_router_address="tcp://127.0.0.1:26100",
|
||||
),
|
||||
)
|
||||
mock_manager_cls = mocker.patch(
|
||||
"vllm_omni.engine.omni_core_engine_proc_manager.OmniCoreEngineProcManager",
|
||||
return_value=engine_manager,
|
||||
)
|
||||
mocker.patch("vllm.v1.engine.utils.CoreEngineProcManager", return_value=engine_manager)
|
||||
mocker.patch("signal.signal")
|
||||
|
||||
run_headless(args)
|
||||
run_headless(_make_headless_args(stage_id=0))
|
||||
|
||||
assert mock_register.call_args.kwargs["replica_id"] == 2
|
||||
# The launcher must request auto-assignment (replica_id=None) and the
|
||||
# full response so it can wire the master-allocated coordinator into the
|
||||
# spawned subprocess. ``replica_binds_sockets=False`` is required for LLM
|
||||
# because the head binds all three sockets (handshake, input, output).
|
||||
assert mock_register.call_count == 1
|
||||
kwargs = mock_register.call_args.kwargs
|
||||
assert kwargs["omni_master_address"] == "127.0.0.1"
|
||||
assert kwargs["omni_master_port"] == 26000
|
||||
assert kwargs["omni_stage_id"] == 0
|
||||
assert kwargs["omni_stage_config"] is stage_cfg
|
||||
assert kwargs["replica_id"] is None
|
||||
assert kwargs["return_full_response"] is True
|
||||
assert kwargs["replica_binds_sockets"] is False
|
||||
|
||||
assert mock_manager_cls.call_count == 1
|
||||
mgr_kwargs = mock_manager_cls.call_args.kwargs
|
||||
assert mgr_kwargs["local_engine_count"] == 1
|
||||
assert mgr_kwargs["local_client"] is False
|
||||
assert mgr_kwargs["handshake_address"] == "tcp://127.0.0.1:26001"
|
||||
assert mgr_kwargs["omni_stage_id"] == 0
|
||||
assert mgr_kwargs["omni_coordinator_address"] == "tcp://127.0.0.1:26100"
|
||||
assert mgr_kwargs["omni_replica_base_id"] == 0
|
||||
|
||||
engine_manager.monitor_engine_liveness.assert_called_once_with()
|
||||
engine_manager.shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_run_headless_launches_diffusion_stage_via_omni_master(mocker: MockerFixture) -> None:
|
||||
args = _make_headless_args()
|
||||
args.replica_id = 1
|
||||
stage_cfg = mocker.Mock(stage_id=3, stage_type="diffusion")
|
||||
stage_cfg.engine_args = mocker.Mock()
|
||||
stage_cfg.engine_input_source = []
|
||||
stage_cfgs = [stage_cfg]
|
||||
metadata = mocker.Mock(stage_id=3, stage_type="diffusion")
|
||||
def test_run_headless_llm_launches_one_manager_per_omni_dp_size_local(mocker: MockerFixture) -> None:
|
||||
"""``--omni-dp-size-local=N`` must spawn N managers, each with its own
|
||||
master-assigned replica_id, and join all of them before returning."""
|
||||
from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse
|
||||
|
||||
stage_cfg = _make_stage_cfg(0, stage_type="llm")
|
||||
parallel_config = SimpleNamespace(
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
data_parallel_rank_local=0,
|
||||
node_rank_within_dp=0,
|
||||
)
|
||||
vllm_config = SimpleNamespace(parallel_config=parallel_config, needs_dp_coordinator=False)
|
||||
manager_a = mocker.Mock()
|
||||
manager_b = mocker.Mock()
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", [stage_cfg]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
return_value=(vllm_config, object),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
side_effect=[
|
||||
StageRegistrationResponse(
|
||||
handshake_address=f"tcp://127.0.0.1:2700{idx}",
|
||||
input_address=f"tcp://127.0.0.1:2710{idx}",
|
||||
output_address=f"tcp://127.0.0.1:2720{idx}",
|
||||
replica_id=idx,
|
||||
coordinator_router_address=None,
|
||||
)
|
||||
for idx in (0, 1)
|
||||
],
|
||||
)
|
||||
mock_manager_cls = mocker.patch(
|
||||
"vllm_omni.engine.omni_core_engine_proc_manager.OmniCoreEngineProcManager",
|
||||
side_effect=[manager_a, manager_b],
|
||||
)
|
||||
mocker.patch("signal.signal")
|
||||
|
||||
run_headless(_make_headless_args(stage_id=0, omni_dp_size_local=2))
|
||||
|
||||
assert mock_manager_cls.call_count == 2
|
||||
assigned_ids = [call.kwargs["omni_replica_base_id"] for call in mock_manager_cls.call_args_list]
|
||||
assert assigned_ids == [0, 1]
|
||||
|
||||
# Multi-replica path joins the monitor threads instead of calling
|
||||
# ``monitor_engine_liveness`` synchronously on the main thread, but every
|
||||
# manager must still be shut down in the finally block.
|
||||
manager_a.shutdown.assert_called_once_with()
|
||||
manager_b.shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_run_headless_diffusion_registers_and_spawns_proc(mocker: MockerFixture) -> None:
|
||||
"""Diffusion headless: registers as auto-assign, spawns a single
|
||||
``StageDiffusionProc`` per local replica, and waits for it via
|
||||
``multiprocessing.connection.wait``."""
|
||||
from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse
|
||||
|
||||
stage_cfg = _make_stage_cfg(1, stage_type="diffusion")
|
||||
od_config = mocker.Mock()
|
||||
proc = mocker.Mock()
|
||||
proc.exitcode = 0
|
||||
proc = mocker.Mock(sentinel=object(), exitcode=0)
|
||||
proc.is_alive.return_value = False
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", stage_cfgs),
|
||||
return_value=("/fake/stages.yaml", [stage_cfg]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=mocker.Mock())
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.extract_stage_metadata", return_value=metadata)
|
||||
mock_inject_stage_info = mocker.patch("vllm_omni.engine.stage_init_utils.inject_kv_stage_info")
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.extract_stage_metadata",
|
||||
return_value=SimpleNamespace(stage_id=1, stage_type="diffusion"),
|
||||
)
|
||||
mock_inject = mocker.patch("vllm_omni.engine.stage_init_utils.inject_kv_stage_info")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_diffusion_config", return_value=od_config)
|
||||
mock_register = mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value=("tcp://127.0.0.1:26001", "tcp://127.0.0.1:26002", "tcp://127.0.0.1:26003"),
|
||||
return_value=StageRegistrationResponse(
|
||||
handshake_address="tcp://127.0.0.1:26001",
|
||||
input_address="tcp://127.0.0.1:26002",
|
||||
output_address="tcp://127.0.0.1:26003",
|
||||
replica_id=0,
|
||||
coordinator_router_address="tcp://127.0.0.1:26100",
|
||||
),
|
||||
)
|
||||
mock_spawn = mocker.patch(
|
||||
"vllm_omni.diffusion.stage_diffusion_proc.spawn_diffusion_proc",
|
||||
return_value=(proc, "tcp://127.0.0.1:26001", "tcp://127.0.0.1:26002", "tcp://127.0.0.1:26003"),
|
||||
return_value=(proc, None, None, None),
|
||||
)
|
||||
mock_handshake = mocker.patch("vllm_omni.diffusion.stage_diffusion_proc.complete_diffusion_handshake")
|
||||
# Replace the blocking wait with one that returns the only proc's sentinel
|
||||
# immediately so the test does not hang.
|
||||
mocker.patch(
|
||||
"multiprocessing.connection.wait",
|
||||
side_effect=lambda sentinels: [sentinels[0]],
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.terminate_alive_proc")
|
||||
mocker.patch("signal.signal")
|
||||
run_headless(args)
|
||||
|
||||
mock_inject_stage_info.assert_called_once_with(stage_cfg, 3)
|
||||
mock_register.assert_called_once_with(
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
omni_stage_id=3,
|
||||
omni_stage_config=stage_cfg,
|
||||
return_addresses=True,
|
||||
replica_id=1,
|
||||
)
|
||||
mock_spawn.assert_called_once_with(
|
||||
"fake-model",
|
||||
od_config,
|
||||
handshake_address="tcp://127.0.0.1:26001",
|
||||
request_address="tcp://127.0.0.1:26002",
|
||||
response_address="tcp://127.0.0.1:26003",
|
||||
)
|
||||
run_headless(_make_headless_args(stage_id=1))
|
||||
|
||||
mock_inject.assert_called_once_with(stage_cfg, 1)
|
||||
|
||||
reg_kwargs = mock_register.call_args.kwargs
|
||||
assert reg_kwargs["omni_master_address"] == "127.0.0.1"
|
||||
assert reg_kwargs["omni_master_port"] == 26000
|
||||
assert reg_kwargs["omni_stage_id"] == 1
|
||||
assert reg_kwargs["omni_stage_config"] is stage_cfg
|
||||
assert reg_kwargs["replica_id"] is None
|
||||
assert reg_kwargs["return_full_response"] is True
|
||||
|
||||
spawn_kwargs = mock_spawn.call_args.kwargs
|
||||
assert spawn_kwargs["handshake_address"] == "tcp://127.0.0.1:26001"
|
||||
assert spawn_kwargs["request_address"] == "tcp://127.0.0.1:26002"
|
||||
assert spawn_kwargs["response_address"] == "tcp://127.0.0.1:26003"
|
||||
assert spawn_kwargs["omni_coordinator_address"] == "tcp://127.0.0.1:26100"
|
||||
assert spawn_kwargs["omni_stage_id"] == 1
|
||||
assert spawn_kwargs["omni_replica_id"] == 0
|
||||
|
||||
mock_handshake.assert_called_once_with(proc, "tcp://127.0.0.1:26001", 600)
|
||||
proc.join.assert_called_once_with()
|
||||
|
||||
|
||||
def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixture) -> None:
|
||||
"""A diffusion replica that exits with a non-zero code must surface as a
|
||||
RuntimeError from ``run_headless`` (the head needs the signal to roll
|
||||
back its own stage init)."""
|
||||
from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse
|
||||
|
||||
stage_cfg = _make_stage_cfg(1, stage_type="diffusion")
|
||||
proc = mocker.Mock(sentinel=object(), exitcode=137, name="proc-stage1-rep0")
|
||||
proc.is_alive.return_value = False
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.load_and_resolve_stage_configs",
|
||||
return_value=("/fake/stages.yaml", [stage_cfg]),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.extract_stage_metadata",
|
||||
return_value=SimpleNamespace(stage_id=1, stage_type="diffusion"),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.inject_kv_stage_info")
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_diffusion_config", return_value=mocker.Mock())
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
return_value=StageRegistrationResponse(
|
||||
handshake_address="tcp://127.0.0.1:26001",
|
||||
input_address="tcp://127.0.0.1:26002",
|
||||
output_address="tcp://127.0.0.1:26003",
|
||||
replica_id=0,
|
||||
coordinator_router_address=None,
|
||||
),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.stage_diffusion_proc.spawn_diffusion_proc",
|
||||
return_value=(proc, None, None, None),
|
||||
)
|
||||
mocker.patch("vllm_omni.diffusion.stage_diffusion_proc.complete_diffusion_handshake")
|
||||
mocker.patch(
|
||||
"multiprocessing.connection.wait",
|
||||
side_effect=lambda sentinels: [sentinels[0]],
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.terminate_alive_proc")
|
||||
mocker.patch("signal.signal")
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"exited with code 137"):
|
||||
run_headless(_make_headless_args(stage_id=1))
|
||||
Reference in new issue
Block a user