[WIP][Bugfix][MiniCPM-o] Fix Full duplex bugs for minicpm-challenge (#5496)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-07-28 15:59:26 +08:00
1 parent fb89ab43ea
commit ffa9a6af69
60 files changed
+3696 -634

No files matched your search

+4 -4
View File
@@ -209,10 +209,10 @@ class AsyncOmniEngine:
self._omni_master_port,
)
# Stage resolution pops deploy_config, so get the pipeline endpoint
# restriction beforehand. TODO (Alex) make this cleaner and refactor
# stage config resolution to remove kwargs hacks.
deploy_config_path = kwargs.get("deploy_config")
# Stage resolution pops deploy_config, so get pipeline-wide settings
# beforehand. The stage CLI exposes the same deploy YAML through
# stage_configs_path.
deploy_config_path = kwargs.get("deploy_config") or kwargs.get("stage_configs_path")
pipeline_config = StageConfigFactory.get_pipeline_config(
model=model,
trust_remote_code=trust_remote_code,
+31 -18
View File
@@ -13,7 +13,7 @@ from __future__ import annotations
import asyncio
import time as _time
from collections.abc import Callable
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
@@ -218,11 +218,18 @@ class _OrchestratorDuplexStagePort:
request_states: dict[str, OrchestratorRequestState],
running_counter: OmniRequestCounter | None,
cleanup_request_ids: Callable[..., Any],
async_chunk: bool,
prewarm_async_chunk_stages: Callable[
[str, Any, OrchestratorRequestState],
Awaitable[None],
],
) -> None:
self._stage_pools = stage_pools
self._request_states = request_states
self._running_counter = running_counter
self._cleanup_request_ids = cleanup_request_ids
self._async_chunk = async_chunk
self._prewarm_async_chunk_stages = prewarm_async_chunk_stages
@property
def stage_count(self) -> int:
@@ -299,6 +306,12 @@ class _OrchestratorDuplexStagePort:
replica_id = await pool.submit_update(context.request_id, request_state, request)
else:
replica_id = await pool.submit_initial(context.request_id, request_state, request, prompt_text=None)
if self._async_chunk and context.stage_id == 0:
await self._prewarm_async_chunk_stages(
context.request_id,
request,
request_state,
)
request_state.duplex_stage_fences[context.stage_id] = context.fence
request_state.stage_submit_ts[context.stage_id] = _time.time()
if not request_state.running_counter_registered and self._running_counter is not None:
@@ -388,6 +401,8 @@ class Orchestrator:
request_states=self.request_states,
running_counter=self._running_counter,
cleanup_request_ids=self._cleanup_request_ids,
async_chunk=self.async_chunk,
prewarm_async_chunk_stages=self._prewarm_async_chunk_stages,
),
result_sink=self.rpc_async_queue,
lifecycle_sink=self.output_async_queue,
@@ -681,24 +696,16 @@ class Orchestrator:
final_stage_id = msg.final_stage_id
req_state = self.request_states.get(request_id)
if req_state is None:
# Streaming updates always follow the first-chunk add_request
# through the same ordered submission queue, so an unknown id
# here can only mean the request already finished or was
# aborted (e.g. the client disconnected). Re-adding it would
# resurrect a headless session that keeps cycling through the
# stages with nobody consuming its outputs (issue #4271).
logger.warning(
"[Orchestrator] streaming_update for unknown req=%s, falling back to add_request",
"[Orchestrator] streaming_update for unknown req=%s; dropping (request finished or aborted)",
request_id,
)
fallback_msg = StageSubmissionMessage(
type="add_request",
request_id=msg.request_id,
prompt=msg.prompt,
original_prompt=msg.original_prompt,
output_prompt_text=msg.output_prompt_text,
sampling_params_list=msg.sampling_params_list,
final_stage_id=msg.final_stage_id,
final_output_stage_ids=msg.final_output_stage_ids,
preprocess_ms=msg.preprocess_ms,
request_timestamp=msg.request_timestamp,
enqueue_ts=msg.enqueue_ts,
)
await self._handle_add_request(fallback_msg)
return
if msg.sampling_params_list:
@@ -2025,7 +2032,7 @@ class Orchestrator:
_t_submit_start = _time.perf_counter()
if next_pool.stage_type == "diffusion":
await next_pool.submit_initial(
replica_id = await next_pool.submit_initial(
request_id,
req_state,
req_state.prompt,
@@ -2064,12 +2071,18 @@ class Orchestrator:
resumable=downstream_resumable,
)
request.external_req_id = request.request_id
await next_pool.submit_initial(
replica_id = await next_pool.submit_initial(
request_id,
req_state,
request,
prompt_text=None,
)
self._record_duplex_stage_submission(
next_stage_id,
request_id,
replica_id,
req_state,
)
# async_chunk pre-submit fires per stage edge (N-1 -> N). Source
# replica is stage 0's bound replica (single-replica thinker in
+175 -69
View File
@@ -183,6 +183,10 @@ class OmniMasterServer:
# seeded with the registration port (and coordinator ROUTER port) that
# are already bound on the same host.
self._allocated_ports: set[int] = set()
# Keep route ports bound until their ZMQ owners are ready to bind them.
# Recording only the integer leaves a race with unrelated startup
# consumers (notably NCCL's own ephemeral-port selection).
self._port_reservations: dict[int, socket.socket] = {}
if master_port:
self._allocated_ports.add(int(master_port))
# Coordinator ROUTER address echoed back in every registration reply
@@ -283,9 +287,11 @@ class OmniMasterServer:
``get_open_ports_list`` dedups only within its own call, so we redraw any
port that collides with one already recorded in ``self._allocated_ports``
(registration/coordinator ports plus every previously-allocated route
port) and register the winners before returning. Callers already hold the
relevant allocation context (``__init__`` is single-threaded;
registration holds ``self._alloc_lock``).
port). Each winner remains OS-bound until its ZMQ owner is ready, so
another startup subsystem cannot claim it in the allocation-to-bind
window. Callers already hold the relevant allocation context
(``__init__`` is single-threaded; registration holds
``self._alloc_lock``).
"""
picked: list[int] = []
# Bounded retry budget so a pathological host (near port exhaustion)
@@ -294,7 +300,17 @@ class OmniMasterServer:
for port in get_open_ports_list(count=count - len(picked)):
if port in self._allocated_ports:
continue
reservation = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# Reserve on every local interface. The advertised address
# can be a routable interface IP, but wildcard binding also
# protects the route from local NCCL/socket allocations.
reservation.bind(("", port))
except OSError:
reservation.close()
continue
self._allocated_ports.add(port)
self._port_reservations[port] = reservation
picked.append(port)
if len(picked) == count:
return picked
@@ -303,6 +319,33 @@ class OmniMasterServer:
f"(host={self._address}); {len(self._allocated_ports)} already in use."
)
def release_route_port_reservations(
self,
stage_id: int,
replica_id: int = 0,
*,
handshake: bool = True,
data: bool = True,
) -> None:
"""Release reserved route ports immediately before their ZMQ binds."""
alloc = self.get_allocation(stage_id, replica_id)
ports: list[int] = []
if handshake:
handshake_port = _port_from_zmq_address(alloc.handshake_bind_address)
if handshake_port is not None:
ports.append(handshake_port)
if data:
for address in (alloc.input_bind_address, alloc.output_bind_address):
port = _port_from_zmq_address(address)
if port is not None:
ports.append(port)
with self._alloc_lock:
for port in ports:
reservation = self._port_reservations.pop(port, None)
if reservation is not None:
reservation.close()
def _next_free_replica_id(self, stage_id: int) -> int:
"""Return the next replica id to assign for an auto-assign registration.
@@ -437,6 +480,11 @@ class OmniMasterServer:
self._stop_event.set()
if self._thread is not None:
self._thread.join(timeout=10)
with self._alloc_lock:
reservations = list(self._port_reservations.values())
self._port_reservations.clear()
for reservation in reservations:
reservation.close()
# ------------------------------------------------------------------
# Internal server logic
@@ -771,20 +819,34 @@ def connect_remote_engine_cores(
handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
yield StageReplicaResources(
coordinator=coordinator,
addresses=addresses,
)
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
vllm_config.parallel_config,
False, # coordinated_dp
vllm_config.cache_config,
None, # proc_manager (remote — no local procs)
None, # coord_process
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=True,
data=False,
)
try:
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
yield StageReplicaResources(
coordinator=coordinator,
addresses=addresses,
)
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
vllm_config.parallel_config,
False, # coordinated_dp
vllm_config.cache_config,
None, # proc_manager (remote — no local procs)
None, # coord_process
)
finally:
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=False,
data=True,
)
@@ -802,17 +864,31 @@ def connect_remote_diffusion_proc(
).handshake_bind_address
logger.info("Waiting for remote diffusion proc for stage %d replica %d", stage_id, replica_id)
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
yield StageReplicaResources(addresses=addresses)
wait_for_engine_startup(
handshake_socket,
addresses,
[CoreEngine(index=0, local=False)],
_single_diffusion_parallel_config(local_client=False),
False,
None,
None,
None,
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=True,
data=False,
)
try:
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
yield StageReplicaResources(addresses=addresses)
wait_for_engine_startup(
handshake_socket,
addresses,
[CoreEngine(index=0, local=False)],
_single_diffusion_parallel_config(local_client=False),
False,
None,
None,
None,
)
finally:
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=False,
data=True,
)
@@ -918,49 +994,63 @@ def _launch_omni_core_engines(
# Bind the pre-allocated handshake socket for this stage.
handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
if omni_coordinator_address is not None:
# Use the omni subclass so each spawned subprocess instantiates
# an OmniCoordClientForStage and heartbeats to the coordinator.
from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=True,
data=False,
)
try:
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
if omni_coordinator_address is not None:
# Use the omni subclass so each spawned subprocess instantiates
# an OmniCoordClientForStage and heartbeats to the coordinator.
from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager(
local_engine_count=local_engine_count,
start_index=start_index,
local_start_index=local_start_index,
vllm_config=vllm_config,
local_client=True,
handshake_address=handshake_address,
executor_class=executor_class,
log_stats=log_stats,
omni_stage_id=stage_id,
omni_coordinator_address=omni_coordinator_address,
omni_replica_base_id=replica_id,
)
else:
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
local_engine_manager = CoreEngineProcManager(
local_engine_count=local_engine_count,
start_index=start_index,
local_start_index=local_start_index,
vllm_config=vllm_config,
local_client=True,
handshake_address=handshake_address,
executor_class=executor_class,
log_stats=log_stats,
)
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager(
local_engine_count=local_engine_count,
start_index=start_index,
local_start_index=local_start_index,
vllm_config=vllm_config,
local_client=True,
handshake_address=handshake_address,
executor_class=executor_class,
log_stats=log_stats,
omni_stage_id=stage_id,
omni_coordinator_address=omni_coordinator_address,
omni_replica_base_id=replica_id,
)
else:
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
local_engine_manager = CoreEngineProcManager(
local_engine_count=local_engine_count,
start_index=start_index,
local_start_index=local_start_index,
vllm_config=vllm_config,
local_client=True,
handshake_address=handshake_address,
executor_class=executor_class,
log_stats=log_stats,
)
yield local_engine_manager, coordinator, addresses
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
parallel_config,
parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe,
vllm_config.cache_config,
local_engine_manager,
coordinator.proc if coordinator else None,
yield local_engine_manager, coordinator, addresses
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
parallel_config,
parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe,
vllm_config.cache_config,
local_engine_manager,
coordinator.proc if coordinator else None,
)
finally:
omni_master_server.release_route_port_reservations(
stage_id,
replica_id,
handshake=False,
data=True,
)
@@ -1448,6 +1538,12 @@ def launch_diffusion_stage_replica(
omni_stage_config=stage_config,
replica_id=replica_id,
)
omni_master_server.release_route_port_reservations(
metadata.stage_id,
replica_id,
handshake=True,
data=False,
)
proc_manager = stage_diffusion_proc.StageDiffusionProcManager(
model=model,
od_config=od_config,
@@ -1461,6 +1557,12 @@ def launch_diffusion_stage_replica(
omni_stage_id=metadata.stage_id,
omni_replica_id=replica_id,
)
omni_master_server.release_route_port_reservations(
metadata.stage_id,
replica_id,
handshake=False,
data=True,
)
client = StageDiffusionClient.from_addresses(
metadata,
request_address=proc_manager.addresses.inputs[0],
@@ -1474,6 +1576,10 @@ def launch_diffusion_stage_replica(
lock_fds=lock_fds,
)
except Exception:
omni_master_server.release_route_port_reservations(
metadata.stage_id,
replica_id,
)
if proc_manager is not None:
proc_manager.shutdown()
if lock_fds: