[WIP][Bugfix][MiniCPM-o] Fix Full duplex bugs for minicpm-challenge (#5496)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
60 files changed
+3696
-634
No files matched your search
@@ -209,10 +209,10 @@ class AsyncOmniEngine:
|
||||
self._omni_master_port,
|
||||
)
|
||||
|
||||
# Stage resolution pops deploy_config, so get the pipeline endpoint
|
||||
# restriction beforehand. TODO (Alex) make this cleaner and refactor
|
||||
# stage config resolution to remove kwargs hacks.
|
||||
deploy_config_path = kwargs.get("deploy_config")
|
||||
# Stage resolution pops deploy_config, so get pipeline-wide settings
|
||||
# beforehand. The stage CLI exposes the same deploy YAML through
|
||||
# stage_configs_path.
|
||||
deploy_config_path = kwargs.get("deploy_config") or kwargs.get("stage_configs_path")
|
||||
pipeline_config = StageConfigFactory.get_pipeline_config(
|
||||
model=model,
|
||||
trust_remote_code=trust_remote_code,
|
||||
|
||||
@@ -13,7 +13,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time as _time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -218,11 +218,18 @@ class _OrchestratorDuplexStagePort:
|
||||
request_states: dict[str, OrchestratorRequestState],
|
||||
running_counter: OmniRequestCounter | None,
|
||||
cleanup_request_ids: Callable[..., Any],
|
||||
async_chunk: bool,
|
||||
prewarm_async_chunk_stages: Callable[
|
||||
[str, Any, OrchestratorRequestState],
|
||||
Awaitable[None],
|
||||
],
|
||||
) -> None:
|
||||
self._stage_pools = stage_pools
|
||||
self._request_states = request_states
|
||||
self._running_counter = running_counter
|
||||
self._cleanup_request_ids = cleanup_request_ids
|
||||
self._async_chunk = async_chunk
|
||||
self._prewarm_async_chunk_stages = prewarm_async_chunk_stages
|
||||
|
||||
@property
|
||||
def stage_count(self) -> int:
|
||||
@@ -299,6 +306,12 @@ class _OrchestratorDuplexStagePort:
|
||||
replica_id = await pool.submit_update(context.request_id, request_state, request)
|
||||
else:
|
||||
replica_id = await pool.submit_initial(context.request_id, request_state, request, prompt_text=None)
|
||||
if self._async_chunk and context.stage_id == 0:
|
||||
await self._prewarm_async_chunk_stages(
|
||||
context.request_id,
|
||||
request,
|
||||
request_state,
|
||||
)
|
||||
request_state.duplex_stage_fences[context.stage_id] = context.fence
|
||||
request_state.stage_submit_ts[context.stage_id] = _time.time()
|
||||
if not request_state.running_counter_registered and self._running_counter is not None:
|
||||
@@ -388,6 +401,8 @@ class Orchestrator:
|
||||
request_states=self.request_states,
|
||||
running_counter=self._running_counter,
|
||||
cleanup_request_ids=self._cleanup_request_ids,
|
||||
async_chunk=self.async_chunk,
|
||||
prewarm_async_chunk_stages=self._prewarm_async_chunk_stages,
|
||||
),
|
||||
result_sink=self.rpc_async_queue,
|
||||
lifecycle_sink=self.output_async_queue,
|
||||
@@ -681,24 +696,16 @@ class Orchestrator:
|
||||
final_stage_id = msg.final_stage_id
|
||||
req_state = self.request_states.get(request_id)
|
||||
if req_state is None:
|
||||
# Streaming updates always follow the first-chunk add_request
|
||||
# through the same ordered submission queue, so an unknown id
|
||||
# here can only mean the request already finished or was
|
||||
# aborted (e.g. the client disconnected). Re-adding it would
|
||||
# resurrect a headless session that keeps cycling through the
|
||||
# stages with nobody consuming its outputs (issue #4271).
|
||||
logger.warning(
|
||||
"[Orchestrator] streaming_update for unknown req=%s, falling back to add_request",
|
||||
"[Orchestrator] streaming_update for unknown req=%s; dropping (request finished or aborted)",
|
||||
request_id,
|
||||
)
|
||||
fallback_msg = StageSubmissionMessage(
|
||||
type="add_request",
|
||||
request_id=msg.request_id,
|
||||
prompt=msg.prompt,
|
||||
original_prompt=msg.original_prompt,
|
||||
output_prompt_text=msg.output_prompt_text,
|
||||
sampling_params_list=msg.sampling_params_list,
|
||||
final_stage_id=msg.final_stage_id,
|
||||
final_output_stage_ids=msg.final_output_stage_ids,
|
||||
preprocess_ms=msg.preprocess_ms,
|
||||
request_timestamp=msg.request_timestamp,
|
||||
enqueue_ts=msg.enqueue_ts,
|
||||
)
|
||||
await self._handle_add_request(fallback_msg)
|
||||
return
|
||||
|
||||
if msg.sampling_params_list:
|
||||
@@ -2025,7 +2032,7 @@ class Orchestrator:
|
||||
_t_submit_start = _time.perf_counter()
|
||||
|
||||
if next_pool.stage_type == "diffusion":
|
||||
await next_pool.submit_initial(
|
||||
replica_id = await next_pool.submit_initial(
|
||||
request_id,
|
||||
req_state,
|
||||
req_state.prompt,
|
||||
@@ -2064,12 +2071,18 @@ class Orchestrator:
|
||||
resumable=downstream_resumable,
|
||||
)
|
||||
request.external_req_id = request.request_id
|
||||
await next_pool.submit_initial(
|
||||
replica_id = await next_pool.submit_initial(
|
||||
request_id,
|
||||
req_state,
|
||||
request,
|
||||
prompt_text=None,
|
||||
)
|
||||
self._record_duplex_stage_submission(
|
||||
next_stage_id,
|
||||
request_id,
|
||||
replica_id,
|
||||
req_state,
|
||||
)
|
||||
|
||||
# async_chunk pre-submit fires per stage edge (N-1 -> N). Source
|
||||
# replica is stage 0's bound replica (single-replica thinker in
|
||||
|
||||
@@ -183,6 +183,10 @@ class OmniMasterServer:
|
||||
# seeded with the registration port (and coordinator ROUTER port) that
|
||||
# are already bound on the same host.
|
||||
self._allocated_ports: set[int] = set()
|
||||
# Keep route ports bound until their ZMQ owners are ready to bind them.
|
||||
# Recording only the integer leaves a race with unrelated startup
|
||||
# consumers (notably NCCL's own ephemeral-port selection).
|
||||
self._port_reservations: dict[int, socket.socket] = {}
|
||||
if master_port:
|
||||
self._allocated_ports.add(int(master_port))
|
||||
# Coordinator ROUTER address echoed back in every registration reply
|
||||
@@ -283,9 +287,11 @@ class OmniMasterServer:
|
||||
``get_open_ports_list`` dedups only within its own call, so we redraw any
|
||||
port that collides with one already recorded in ``self._allocated_ports``
|
||||
(registration/coordinator ports plus every previously-allocated route
|
||||
port) and register the winners before returning. Callers already hold the
|
||||
relevant allocation context (``__init__`` is single-threaded;
|
||||
registration holds ``self._alloc_lock``).
|
||||
port). Each winner remains OS-bound until its ZMQ owner is ready, so
|
||||
another startup subsystem cannot claim it in the allocation-to-bind
|
||||
window. Callers already hold the relevant allocation context
|
||||
(``__init__`` is single-threaded; registration holds
|
||||
``self._alloc_lock``).
|
||||
"""
|
||||
picked: list[int] = []
|
||||
# Bounded retry budget so a pathological host (near port exhaustion)
|
||||
@@ -294,7 +300,17 @@ class OmniMasterServer:
|
||||
for port in get_open_ports_list(count=count - len(picked)):
|
||||
if port in self._allocated_ports:
|
||||
continue
|
||||
reservation = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
try:
|
||||
# Reserve on every local interface. The advertised address
|
||||
# can be a routable interface IP, but wildcard binding also
|
||||
# protects the route from local NCCL/socket allocations.
|
||||
reservation.bind(("", port))
|
||||
except OSError:
|
||||
reservation.close()
|
||||
continue
|
||||
self._allocated_ports.add(port)
|
||||
self._port_reservations[port] = reservation
|
||||
picked.append(port)
|
||||
if len(picked) == count:
|
||||
return picked
|
||||
@@ -303,6 +319,33 @@ class OmniMasterServer:
|
||||
f"(host={self._address}); {len(self._allocated_ports)} already in use."
|
||||
)
|
||||
|
||||
def release_route_port_reservations(
|
||||
self,
|
||||
stage_id: int,
|
||||
replica_id: int = 0,
|
||||
*,
|
||||
handshake: bool = True,
|
||||
data: bool = True,
|
||||
) -> None:
|
||||
"""Release reserved route ports immediately before their ZMQ binds."""
|
||||
alloc = self.get_allocation(stage_id, replica_id)
|
||||
ports: list[int] = []
|
||||
if handshake:
|
||||
handshake_port = _port_from_zmq_address(alloc.handshake_bind_address)
|
||||
if handshake_port is not None:
|
||||
ports.append(handshake_port)
|
||||
if data:
|
||||
for address in (alloc.input_bind_address, alloc.output_bind_address):
|
||||
port = _port_from_zmq_address(address)
|
||||
if port is not None:
|
||||
ports.append(port)
|
||||
|
||||
with self._alloc_lock:
|
||||
for port in ports:
|
||||
reservation = self._port_reservations.pop(port, None)
|
||||
if reservation is not None:
|
||||
reservation.close()
|
||||
|
||||
def _next_free_replica_id(self, stage_id: int) -> int:
|
||||
"""Return the next replica id to assign for an auto-assign registration.
|
||||
|
||||
@@ -437,6 +480,11 @@ class OmniMasterServer:
|
||||
self._stop_event.set()
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=10)
|
||||
with self._alloc_lock:
|
||||
reservations = list(self._port_reservations.values())
|
||||
self._port_reservations.clear()
|
||||
for reservation in reservations:
|
||||
reservation.close()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal server logic
|
||||
@@ -771,20 +819,34 @@ def connect_remote_engine_cores(
|
||||
|
||||
handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address
|
||||
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
yield StageReplicaResources(
|
||||
coordinator=coordinator,
|
||||
addresses=addresses,
|
||||
)
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
vllm_config.parallel_config,
|
||||
False, # coordinated_dp
|
||||
vllm_config.cache_config,
|
||||
None, # proc_manager (remote — no local procs)
|
||||
None, # coord_process
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=True,
|
||||
data=False,
|
||||
)
|
||||
try:
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
yield StageReplicaResources(
|
||||
coordinator=coordinator,
|
||||
addresses=addresses,
|
||||
)
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
vllm_config.parallel_config,
|
||||
False, # coordinated_dp
|
||||
vllm_config.cache_config,
|
||||
None, # proc_manager (remote — no local procs)
|
||||
None, # coord_process
|
||||
)
|
||||
finally:
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=False,
|
||||
data=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -802,17 +864,31 @@ def connect_remote_diffusion_proc(
|
||||
).handshake_bind_address
|
||||
|
||||
logger.info("Waiting for remote diffusion proc for stage %d replica %d", stage_id, replica_id)
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
yield StageReplicaResources(addresses=addresses)
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
[CoreEngine(index=0, local=False)],
|
||||
_single_diffusion_parallel_config(local_client=False),
|
||||
False,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=True,
|
||||
data=False,
|
||||
)
|
||||
try:
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
yield StageReplicaResources(addresses=addresses)
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
[CoreEngine(index=0, local=False)],
|
||||
_single_diffusion_parallel_config(local_client=False),
|
||||
False,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
finally:
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=False,
|
||||
data=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -918,49 +994,63 @@ def _launch_omni_core_engines(
|
||||
# Bind the pre-allocated handshake socket for this stage.
|
||||
handshake_bind_address = omni_master_server.get_allocation(stage_id, replica_id=replica_id).handshake_bind_address
|
||||
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
if omni_coordinator_address is not None:
|
||||
# Use the omni subclass so each spawned subprocess instantiates
|
||||
# an OmniCoordClientForStage and heartbeats to the coordinator.
|
||||
from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=True,
|
||||
data=False,
|
||||
)
|
||||
try:
|
||||
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
|
||||
if omni_coordinator_address is not None:
|
||||
# Use the omni subclass so each spawned subprocess instantiates
|
||||
# an OmniCoordClientForStage and heartbeats to the coordinator.
|
||||
from vllm_omni.engine.stage_engine_core_proc_manager import StageEngineCoreProcManager
|
||||
|
||||
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
|
||||
local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager(
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index,
|
||||
vllm_config=vllm_config,
|
||||
local_client=True,
|
||||
handshake_address=handshake_address,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
omni_stage_id=stage_id,
|
||||
omni_coordinator_address=omni_coordinator_address,
|
||||
omni_replica_base_id=replica_id,
|
||||
)
|
||||
else:
|
||||
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
|
||||
local_engine_manager = CoreEngineProcManager(
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index,
|
||||
vllm_config=vllm_config,
|
||||
local_client=True,
|
||||
handshake_address=handshake_address,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
|
||||
local_engine_manager: CoreEngineProcManager = StageEngineCoreProcManager(
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index,
|
||||
vllm_config=vllm_config,
|
||||
local_client=True,
|
||||
handshake_address=handshake_address,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
omni_stage_id=stage_id,
|
||||
omni_coordinator_address=omni_coordinator_address,
|
||||
omni_replica_base_id=replica_id,
|
||||
)
|
||||
else:
|
||||
with scoped_spawn_device_env(stage_visible_devices, spawn_device_lock):
|
||||
local_engine_manager = CoreEngineProcManager(
|
||||
local_engine_count=local_engine_count,
|
||||
start_index=start_index,
|
||||
local_start_index=local_start_index,
|
||||
vllm_config=vllm_config,
|
||||
local_client=True,
|
||||
handshake_address=handshake_address,
|
||||
executor_class=executor_class,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
yield local_engine_manager, coordinator, addresses
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
parallel_config,
|
||||
parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe,
|
||||
vllm_config.cache_config,
|
||||
local_engine_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
yield local_engine_manager, coordinator, addresses
|
||||
wait_for_engine_startup(
|
||||
handshake_socket,
|
||||
addresses,
|
||||
engines_to_handshake,
|
||||
parallel_config,
|
||||
parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe,
|
||||
vllm_config.cache_config,
|
||||
local_engine_manager,
|
||||
coordinator.proc if coordinator else None,
|
||||
)
|
||||
finally:
|
||||
omni_master_server.release_route_port_reservations(
|
||||
stage_id,
|
||||
replica_id,
|
||||
handshake=False,
|
||||
data=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -1448,6 +1538,12 @@ def launch_diffusion_stage_replica(
|
||||
omni_stage_config=stage_config,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
omni_master_server.release_route_port_reservations(
|
||||
metadata.stage_id,
|
||||
replica_id,
|
||||
handshake=True,
|
||||
data=False,
|
||||
)
|
||||
proc_manager = stage_diffusion_proc.StageDiffusionProcManager(
|
||||
model=model,
|
||||
od_config=od_config,
|
||||
@@ -1461,6 +1557,12 @@ def launch_diffusion_stage_replica(
|
||||
omni_stage_id=metadata.stage_id,
|
||||
omni_replica_id=replica_id,
|
||||
)
|
||||
omni_master_server.release_route_port_reservations(
|
||||
metadata.stage_id,
|
||||
replica_id,
|
||||
handshake=False,
|
||||
data=True,
|
||||
)
|
||||
client = StageDiffusionClient.from_addresses(
|
||||
metadata,
|
||||
request_address=proc_manager.addresses.inputs[0],
|
||||
@@ -1474,6 +1576,10 @@ def launch_diffusion_stage_replica(
|
||||
lock_fds=lock_fds,
|
||||
)
|
||||
except Exception:
|
||||
omni_master_server.release_route_port_reservations(
|
||||
metadata.stage_id,
|
||||
replica_id,
|
||||
)
|
||||
if proc_manager is not None:
|
||||
proc_manager.shutdown()
|
||||
if lock_fds:
|
||||
|
||||
Reference in new issue
Block a user