fix(engine): dedup ZMQ ports across stages to stop OmniMasterServer bind races

Each (stage_id, replica_id) route allocated its 3 handshake/input/output
ports via a separate get_open_ports_list(count=3) call. That helper only
guarantees uniqueness *within* a single call, so across routes the OS could
hand back a just-closed ephemeral port. Two stages then drew the same port
and the second engine subprocess to bind() it died with
"zmq.error.ZMQError: Address already in use", surfacing as
"Orchestrator initialization failed" / "Stage N replica M exited before API
server became ready". This flaked multi-stage startup (e.g. Qwen3-Omni
thinker/talker/code2wav) — one CI shard passed while a parallel shard failed
on the same commit (build 2630, dev/vllm-align).

OmniMasterServer now tracks every port it hands out in self._allocated_ports
(seeded with the registration port and the coordinator ROUTER port) and a new
_alloc_unique_ports() redraws any collision with a bounded retry budget,
raising loudly on port exhaustion instead of spinning. Adds _port_from_zmq_address()
helper and two CPU-only regression tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Opus 4.8 committed 2026-07-11 14:21:17 +08:00
1 parent cbc3b5e923
commit c898701d53
2 files changed
+108 -1

No files matched your search

@@ -874,3 +874,54 @@ def test_build_engine_args_dict_uses_diffusion_attention_config_key():
assert "attention_config" not in engine_args_dict
assert engine_args_dict["diffusion_attention_config"].default.backend == "FLASH_ATTN"
def test_omni_master_server_allocates_globally_unique_route_ports(monkeypatch):
"""Regression: two stages must never draw the same ZMQ port.
``get_open_ports_list`` only dedups within a single call, so per-route
allocation used to let a later stage reuse an earlier stage's port. The
second engine to ``bind()`` then died with ``zmq.error.ZMQError: Address
already in use`` (flaky multi-stage startup, e.g. Qwen3-Omni thinker/talker/
code2wav). ``OmniMasterServer`` now dedups every port it hands out.
"""
import itertools
from vllm_omni.engine import stage_engine_startup as ses
# A colliding prefix (repeats + the master port 9000) followed by an endless
# fresh stream, so the only way to succeed is to redraw the collisions.
supply = itertools.chain([9000, 9000, 9001, 9001, 9000], itertools.count(9002))
def fake_get_open_ports_list(count):
return [next(supply) for _ in range(count)]
monkeypatch.setattr(ses, "get_open_ports_list", fake_get_open_ports_list)
server = ses.OmniMasterServer(
master_address="127.0.0.1",
master_port=9000, # seed: a route must not reuse the registration port
stage_ids=[0, 1, 2],
)
ports = []
for sid in (0, 1, 2):
alloc = server.get_allocation(sid)
for addr in (
alloc.handshake_bind_address,
alloc.input_bind_address,
alloc.output_bind_address,
):
ports.append(ses._port_from_zmq_address(addr))
assert len(ports) == len(set(ports)), f"duplicate route ports allocated: {ports}"
assert 9000 not in ports, "route reused the master registration port"
def test_port_from_zmq_address_parsing():
from vllm_omni.engine.stage_engine_startup import _port_from_zmq_address
assert _port_from_zmq_address("tcp://127.0.0.1:34277") == 34277
assert _port_from_zmq_address(None) is None
assert _port_from_zmq_address("ipc:///tmp/sock") is None
assert _port_from_zmq_address("tcp://host:not-a-port") is None