[core]refactor communication layer: PR1(Added Refactor Infra Only) (#1555)
Signed-off-by: natureofnature <wzliu@connect.hku.hk> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
2b70e89535
commit
0d4e975e1b
9 files changed
+4712
-3
No files matched your search
@@ -0,0 +1,380 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduling-side coordination for chunk and full_payload input waiting.
|
||||
|
||||
Manages WAITING_FOR_CHUNK and WAITING_FOR_INPUT state transitions
|
||||
based on readiness signals from OmniConnectorOutput, without ever
|
||||
calling connector.put()/get().
|
||||
|
||||
This replaces the scheduling half of OmniChunkTransferAdapter; the
|
||||
transport half lives in OmniConnectorModelRunnerMixin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OmniSchedulingCoordinator:
|
||||
"""Pure-scheduling coordinator for chunk and full_payload input waiting.
|
||||
|
||||
The Scheduler owns an instance of this class. It consumes readiness
|
||||
signals produced by the Model Runner's ``OmniConnectorModelRunnerMixin``
|
||||
(via ``OmniConnectorOutput``) and manages ``WAITING_FOR_CHUNK`` and
|
||||
``WAITING_FOR_INPUT`` state transitions accordingly.
|
||||
"""
|
||||
|
||||
def __init__(self, scheduler_max_num_seqs: int, stage_id: int = 0, async_chunk: bool = False):
|
||||
self._stage_id = stage_id
|
||||
self._scheduler_max_num_seqs = scheduler_max_num_seqs
|
||||
self._async_chunk = async_chunk
|
||||
|
||||
self.finished_requests: set[str] = set()
|
||||
self.requests_with_ready_chunks: set[str] = set()
|
||||
self._full_payload_input_received: set[str] = set()
|
||||
|
||||
self._waiting_for_chunk_waiting: deque[Any] = deque()
|
||||
self._waiting_for_chunk_running: deque[Any] = deque()
|
||||
|
||||
# Request IDs that were newly registered for chunk recv this cycle.
|
||||
# The engine/Model Runner should call register_chunk_recv() for these
|
||||
# so the bg thread starts polling.
|
||||
self.pending_chunk_registrations: list[Any] = []
|
||||
|
||||
# Requests waiting for full_payload stage input (WAITING_FOR_INPUT).
|
||||
self._waiting_for_input: deque[Any] = deque()
|
||||
self.pending_input_registrations: list[Any] = []
|
||||
|
||||
# Monotonic timestamp recording when each request first entered
|
||||
# WAITING_FOR_CHUNK or WAITING_FOR_INPUT. Used by
|
||||
# collect_timed_out_request_ids() to detect orphaned waits.
|
||||
self._waiting_since: dict[str, float] = {}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Core scheduling methods
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def process_pending_chunks(
|
||||
self,
|
||||
waiting_queue: Any,
|
||||
running_queue: list[Request],
|
||||
chunk_ready_req_ids: set[str],
|
||||
chunk_finished_req_ids: set[str],
|
||||
) -> None:
|
||||
"""Transition requests whose chunks have arrived.
|
||||
|
||||
Args:
|
||||
waiting_queue: Scheduler's waiting request queue.
|
||||
running_queue: Scheduler's running request list.
|
||||
chunk_ready_req_ids: IDs with a newly arrived chunk this cycle.
|
||||
chunk_finished_req_ids: IDs whose final chunk has arrived.
|
||||
"""
|
||||
if self._stage_id == 0 or not self._async_chunk:
|
||||
return
|
||||
|
||||
terminal_ready_req_ids = chunk_ready_req_ids.intersection(chunk_finished_req_ids)
|
||||
self.finished_requests.update(chunk_finished_req_ids - terminal_ready_req_ids)
|
||||
self.pending_chunk_registrations = []
|
||||
|
||||
self._process_chunk_queue(
|
||||
waiting_queue,
|
||||
self._waiting_for_chunk_waiting,
|
||||
RequestStatus.WAITING,
|
||||
chunk_ready_req_ids,
|
||||
)
|
||||
self._process_chunk_queue(
|
||||
running_queue,
|
||||
self._waiting_for_chunk_running,
|
||||
RequestStatus.RUNNING,
|
||||
chunk_ready_req_ids,
|
||||
)
|
||||
self.finished_requests.update(terminal_ready_req_ids)
|
||||
|
||||
while len(running_queue) > self._scheduler_max_num_seqs:
|
||||
request = running_queue.pop()
|
||||
# Must reset status to WAITING so the scheduler treats it as
|
||||
# schedulable work. KV blocks are NOT freed here (unlike a
|
||||
# real preemption), so PREEMPTED would be incorrect.
|
||||
request.status = RequestStatus.WAITING
|
||||
waiting_queue.prepend_requests([request])
|
||||
|
||||
def process_pending_full_payload_inputs(
|
||||
self,
|
||||
waiting_queue: Any,
|
||||
running_queue: list[Request],
|
||||
stage_recv_req_ids: set[str],
|
||||
) -> None:
|
||||
"""Manage WAITING_FOR_INPUT lifecycle for full_payload_mode.
|
||||
|
||||
For non-Stage-0 stages in full_payload_mode (``async_chunk=False``):
|
||||
1. Fresh WAITING requests are transitioned to WAITING_FOR_INPUT
|
||||
and registered for bg-thread polling.
|
||||
2. WAITING_FOR_INPUT requests whose data has arrived (in
|
||||
``stage_recv_req_ids``) are transitioned back to WAITING.
|
||||
"""
|
||||
if self._stage_id == 0:
|
||||
return
|
||||
|
||||
self._full_payload_input_received.update(stage_recv_req_ids)
|
||||
if not self._async_chunk and stage_recv_req_ids:
|
||||
self.finished_requests.update(stage_recv_req_ids)
|
||||
logger.debug(
|
||||
"[Coordinator stage-%s] full_payload recv -> finished_requests: %s",
|
||||
self._stage_id,
|
||||
stage_recv_req_ids,
|
||||
)
|
||||
self.pending_input_registrations = []
|
||||
|
||||
remaining: deque[Any] = deque()
|
||||
for request in self._waiting_for_input:
|
||||
if request.request_id in stage_recv_req_ids:
|
||||
request.status = RequestStatus.WAITING
|
||||
self._waiting_since.pop(request.request_id, None)
|
||||
waiting_queue.add_request(request)
|
||||
else:
|
||||
remaining.append(request)
|
||||
self._waiting_for_input = remaining
|
||||
|
||||
if not self._async_chunk:
|
||||
to_remove: list[Any] = []
|
||||
queue_snapshot = list(waiting_queue)
|
||||
for request in queue_snapshot:
|
||||
if request.status == RequestStatus.WAITING:
|
||||
if request.request_id in self._full_payload_input_received:
|
||||
continue
|
||||
if request.request_id in self.requests_with_ready_chunks:
|
||||
continue
|
||||
if request.request_id in self.finished_requests:
|
||||
continue
|
||||
request.status = RequestStatus.WAITING_FOR_INPUT
|
||||
self._waiting_since.setdefault(request.request_id, time.monotonic())
|
||||
to_remove.append(request)
|
||||
self._waiting_for_input.append(request)
|
||||
self.pending_input_registrations.append(request)
|
||||
elif request.status == RequestStatus.WAITING_FOR_INPUT:
|
||||
if request.request_id in stage_recv_req_ids:
|
||||
request.status = RequestStatus.WAITING
|
||||
self._waiting_since.pop(request.request_id, None)
|
||||
else:
|
||||
to_remove.append(request)
|
||||
self._waiting_for_input.append(request)
|
||||
self.pending_input_registrations.append(request)
|
||||
for request in to_remove:
|
||||
waiting_queue.remove(request)
|
||||
|
||||
def process_pending_full_payload_inputs_legacy(
|
||||
self,
|
||||
waiting_queue: Any,
|
||||
running_queue: list[Request],
|
||||
stage_recv_req_ids: set[str],
|
||||
) -> None:
|
||||
"""Compatibility wrapper for ``process_pending_full_payload_inputs``."""
|
||||
self.process_pending_full_payload_inputs(waiting_queue, running_queue, stage_recv_req_ids)
|
||||
|
||||
def free_finished_request(self, request_id: str) -> None:
|
||||
"""Prune internal tracking sets for a freed request to prevent unbounded growth."""
|
||||
self._full_payload_input_received.discard(request_id)
|
||||
self.finished_requests.discard(request_id)
|
||||
self.requests_with_ready_chunks.discard(request_id)
|
||||
self._waiting_since.pop(request_id, None)
|
||||
|
||||
def collect_timed_out_request_ids(
|
||||
self,
|
||||
timeout_s: float,
|
||||
) -> set[str]:
|
||||
"""Return IDs of requests that have been waiting longer than *timeout_s*.
|
||||
|
||||
Uses ``_waiting_since`` timestamps (always up-to-date) to detect
|
||||
timed-out requests. This method is safe to call at any point in
|
||||
the scheduling cycle — it does **not** rely on coordinator internal
|
||||
queues (which are empty after ``restore_queues()``).
|
||||
|
||||
Clears ``_waiting_since`` for timed-out IDs and defensively removes
|
||||
them from coordinator internal queues if present. The caller
|
||||
(scheduler) should then remove the requests from its queues,
|
||||
set ``FINISHED_ERROR``, and call ``_free_request()`` so that
|
||||
``cleanup_finished_request()`` fires in the model runner mixin.
|
||||
"""
|
||||
if timeout_s <= 0:
|
||||
return set()
|
||||
now = time.monotonic()
|
||||
timed_out_ids: set[str] = set()
|
||||
for req_id, start_time in self._waiting_since.items():
|
||||
if now - start_time > timeout_s:
|
||||
timed_out_ids.add(req_id)
|
||||
if not timed_out_ids:
|
||||
return set()
|
||||
|
||||
# Defensively remove from coordinator internal queues (may already
|
||||
# be empty if restore_queues() has run).
|
||||
for queue_attr in (
|
||||
"_waiting_for_chunk_waiting",
|
||||
"_waiting_for_chunk_running",
|
||||
"_waiting_for_input",
|
||||
):
|
||||
queue = getattr(self, queue_attr)
|
||||
remaining: deque[Any] = deque()
|
||||
for request in queue:
|
||||
if request.request_id not in timed_out_ids:
|
||||
remaining.append(request)
|
||||
setattr(self, queue_attr, remaining)
|
||||
|
||||
for req_id in timed_out_ids:
|
||||
self._waiting_since.pop(req_id, None)
|
||||
logger.warning(
|
||||
"[Coordinator stage-%s] Request %s timed out waiting for chunk/input (waited > %.0fs)",
|
||||
self._stage_id,
|
||||
req_id,
|
||||
timeout_s,
|
||||
)
|
||||
|
||||
return timed_out_ids
|
||||
|
||||
def restore_queues(
|
||||
self,
|
||||
waiting_queue: Any,
|
||||
running_queue: list[Request],
|
||||
) -> None:
|
||||
"""Return waiting-for-chunk/input requests to scheduling queues."""
|
||||
for request in self._waiting_for_chunk_waiting:
|
||||
waiting_queue.add_request(request)
|
||||
self._waiting_for_chunk_waiting = deque()
|
||||
|
||||
if self._waiting_for_chunk_running:
|
||||
running_queue.extend(self._waiting_for_chunk_running)
|
||||
self._waiting_for_chunk_running = deque()
|
||||
|
||||
for request in self._waiting_for_input:
|
||||
waiting_queue.add_request(request)
|
||||
self._waiting_for_input = deque()
|
||||
|
||||
def update_request_metadata(
|
||||
self,
|
||||
requests: dict[str, Request],
|
||||
request_metadata: dict[str, dict[str, Any]],
|
||||
model_mode: str = "ar",
|
||||
) -> None:
|
||||
"""Apply received scheduling metadata to request objects.
|
||||
|
||||
For AR mode: only scheduler-visible metadata is applied locally.
|
||||
For Generation mode: updates ``request.prompt_token_ids``.
|
||||
|
||||
Additionally, if the payload contains ``next_stage_prompt_len``,
|
||||
updates the request's ``prompt_token_ids`` to the correct length.
|
||||
"""
|
||||
for req_id, metadata in request_metadata.items():
|
||||
request = requests.get(req_id)
|
||||
if request is None:
|
||||
continue
|
||||
|
||||
# Handle next_stage_prompt_len if present (for models like Qwen3-Omni).
|
||||
# Only apply when the request has not started decoding yet
|
||||
# (no output tokens). Resetting a mid-decode request would
|
||||
# destroy generated tokens and desync KV cache state.
|
||||
if "next_stage_prompt_len" in metadata:
|
||||
next_len = metadata["next_stage_prompt_len"]
|
||||
if isinstance(next_len, int) and next_len > 0:
|
||||
output_token_ids = getattr(request, "_output_token_ids", None)
|
||||
has_decode_output = output_token_ids is not None and len(output_token_ids) > 0
|
||||
if has_decode_output:
|
||||
logger.debug(
|
||||
"[Coordinator stage-%s] Skipping prompt resize for req %s: "
|
||||
"request already has %s output tokens",
|
||||
self._stage_id,
|
||||
req_id,
|
||||
len(output_token_ids),
|
||||
)
|
||||
else:
|
||||
current_prompt_ids = getattr(request, "prompt_token_ids", []) or []
|
||||
current_prompt_len = len(current_prompt_ids)
|
||||
if current_prompt_len != next_len or getattr(request, "num_prompt_tokens", None) != next_len:
|
||||
new_prompt = [0] * next_len
|
||||
request.prompt_token_ids = new_prompt
|
||||
request.num_prompt_tokens = next_len
|
||||
request._all_token_ids.clear()
|
||||
request._all_token_ids.extend(new_prompt)
|
||||
request._output_token_ids.clear()
|
||||
request.num_computed_tokens = 0
|
||||
logger.debug(
|
||||
"[Coordinator stage-%s] Updated prompt_token_ids length to %s for req %s",
|
||||
self._stage_id,
|
||||
next_len,
|
||||
req_id,
|
||||
)
|
||||
|
||||
if model_mode != "ar":
|
||||
new_ids = metadata.get("code_predictor_codes", [])
|
||||
runtime_seed = None
|
||||
if "left_context_size" in metadata:
|
||||
runtime_seed = {
|
||||
"left_context_size": metadata["left_context_size"],
|
||||
}
|
||||
request._omni_initial_model_buffer = runtime_seed
|
||||
if new_ids:
|
||||
request.prompt_token_ids = new_ids
|
||||
request.num_computed_tokens = 0
|
||||
|
||||
def postprocess_scheduler_output(
|
||||
self,
|
||||
scheduler_output: Any,
|
||||
requests: dict[str, Request] | None = None,
|
||||
) -> None:
|
||||
"""Clear per-cycle ready state after scheduler output is materialized."""
|
||||
self._clear_chunk_ready(scheduler_output)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _process_chunk_queue(
|
||||
self,
|
||||
queue: Any,
|
||||
waiting_for_chunk_list: deque[Any],
|
||||
target_status: RequestStatus,
|
||||
chunk_ready_req_ids: set[str],
|
||||
) -> None:
|
||||
queue_snapshot = list(queue)
|
||||
for request in queue_snapshot:
|
||||
if request.status != RequestStatus.WAITING_FOR_CHUNK:
|
||||
if request.request_id in self.requests_with_ready_chunks:
|
||||
continue
|
||||
if request.request_id in self.finished_requests:
|
||||
continue
|
||||
if request.status == RequestStatus.WAITING_FOR_INPUT:
|
||||
continue
|
||||
if request.request_id in chunk_ready_req_ids:
|
||||
self.requests_with_ready_chunks.add(request.request_id)
|
||||
continue
|
||||
self.pending_chunk_registrations.append(request)
|
||||
request.status = RequestStatus.WAITING_FOR_CHUNK
|
||||
self._waiting_since.setdefault(request.request_id, time.monotonic())
|
||||
else:
|
||||
if request.request_id in chunk_ready_req_ids:
|
||||
request.status = target_status
|
||||
self.requests_with_ready_chunks.add(request.request_id)
|
||||
self._waiting_since.pop(request.request_id, None)
|
||||
continue
|
||||
queue.remove(request)
|
||||
waiting_for_chunk_list.append(request)
|
||||
|
||||
def _clear_chunk_ready(self, scheduler_output: Any) -> None:
|
||||
if scheduler_output.scheduled_new_reqs:
|
||||
for req_data in scheduler_output.scheduled_new_reqs:
|
||||
self.requests_with_ready_chunks.discard(
|
||||
getattr(req_data, "req_id", None),
|
||||
)
|
||||
|
||||
if scheduler_output.scheduled_cached_reqs:
|
||||
for req_id in scheduler_output.scheduled_cached_reqs.req_ids:
|
||||
self.requests_with_ready_chunks.discard(req_id)
|
||||
|
||||
|
||||
# Backward-compatible alias
|
||||
ChunkSchedulingCoordinator = OmniSchedulingCoordinator
|
||||
Reference in new issue
Block a user