249 lines
10 KiB
Python
249 lines
10 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from dataclasses import fields
|
|
|
|
from vllm.logger import init_logger
|
|
|
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
|
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
|
from vllm_omni.diffusion.sched.interface import (
|
|
CachedRequestData,
|
|
DiffusionRequestState,
|
|
DiffusionRequestStatus,
|
|
DiffusionSchedulerOutput,
|
|
NewRequestData,
|
|
SamplingParamsKey,
|
|
SchedulerInterface,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# LoRA identity is derived from `sampling.lora_request`, not a same-named field
|
|
# on sampling params, so it must be resolved separately from the bulk lookup.
|
|
_KEY_FIELD_NAMES = frozenset(f.name for f in fields(SamplingParamsKey)) - {"lora_int_id"}
|
|
|
|
|
|
def get_sampling_params_key(request: OmniDiffusionRequest) -> SamplingParamsKey | None:
|
|
"""Build a batch-compatibility key from the request's sampling params."""
|
|
if len(request.prompts) != 1:
|
|
return None
|
|
|
|
sampling = request.sampling_params
|
|
lora_request = getattr(sampling, "lora_request", None)
|
|
return SamplingParamsKey(
|
|
lora_int_id=lora_request.lora_int_id if lora_request is not None else None,
|
|
**{name: getattr(sampling, name) for name in _KEY_FIELD_NAMES},
|
|
)
|
|
|
|
|
|
class _BaseScheduler(SchedulerInterface):
|
|
"""Shared queue/state bookkeeping for diffusion schedulers."""
|
|
|
|
def __init__(self) -> None:
|
|
self.od_config: OmniDiffusionConfig | None = None
|
|
self._request_states: dict[str, DiffusionRequestState] = {}
|
|
self._step_id: int = 0
|
|
self._waiting: deque[str] = deque()
|
|
self._running: list[str] = []
|
|
self._running_sampling_params_key: SamplingParamsKey | None = None
|
|
self._finished_req_ids: set[str] = set()
|
|
self.max_num_running_reqs: int = 1
|
|
|
|
def initialize(self, od_config: OmniDiffusionConfig) -> None:
|
|
self.od_config = od_config
|
|
self._request_states.clear()
|
|
self._step_id = 0
|
|
self._waiting.clear()
|
|
self._running.clear()
|
|
self._running_sampling_params_key = None
|
|
self._finished_req_ids.clear()
|
|
max_num_seqs = getattr(od_config, "max_num_seqs", 1)
|
|
try:
|
|
self.max_num_running_reqs = max(1, int(max_num_seqs))
|
|
except (TypeError, ValueError):
|
|
self.max_num_running_reqs = 1
|
|
self._reset_scheduler_state()
|
|
|
|
def add_request(self, request: OmniDiffusionRequest) -> str:
|
|
return self._add_request_with_request_id(request.request_id, request)
|
|
|
|
def _add_request_with_request_id(self, request_id: str, request: OmniDiffusionRequest) -> str:
|
|
if request_id in self._request_states:
|
|
raise ValueError(f"request_id {request_id!r} is already active.")
|
|
state = self._make_request_state(request_id, request)
|
|
self._request_states[request_id] = state
|
|
self._waiting.append(request_id)
|
|
logger.debug("%s add_request: %s (waiting=%d)", self.__class__.__name__, request_id, len(self._waiting))
|
|
return request_id
|
|
|
|
def schedule(self) -> DiffusionSchedulerOutput:
|
|
scheduled_new_reqs: list[NewRequestData] = []
|
|
scheduled_cached_request_ids: list[str] = []
|
|
|
|
# First, schedule the RUNNING request(s)
|
|
for request_id in self._running:
|
|
state = self._request_states.get(request_id)
|
|
if state is not None:
|
|
scheduled_cached_request_ids.append(request_id)
|
|
|
|
# Second, schedule WAITING requests while capacity remains.
|
|
while self._waiting and len(self._running) < self.max_num_running_reqs:
|
|
request_id = self._waiting[0]
|
|
state = self._request_states.get(request_id)
|
|
if state is None:
|
|
self._waiting.popleft()
|
|
continue
|
|
if not self._can_schedule_waiting(state):
|
|
break
|
|
|
|
self._waiting.popleft()
|
|
was_new_request = state.status == DiffusionRequestStatus.WAITING
|
|
if not self._running:
|
|
self._running_sampling_params_key = state.sampling_params_key
|
|
state.status = DiffusionRequestStatus.RUNNING
|
|
self._running.append(request_id)
|
|
if was_new_request:
|
|
scheduled_new_reqs.append(NewRequestData.from_state(state))
|
|
else:
|
|
scheduled_cached_request_ids.append(request_id)
|
|
|
|
scheduler_output = DiffusionSchedulerOutput(
|
|
step_id=self._step_id,
|
|
scheduled_new_reqs=scheduled_new_reqs,
|
|
scheduled_cached_reqs=CachedRequestData(request_ids=scheduled_cached_request_ids),
|
|
finished_req_ids=set(self._finished_req_ids),
|
|
num_running_reqs=len(self._running),
|
|
num_waiting_reqs=len(self._waiting),
|
|
)
|
|
|
|
# update after schedule
|
|
self._step_id += 1
|
|
self._finished_req_ids.clear()
|
|
return scheduler_output
|
|
|
|
def has_requests(self) -> bool:
|
|
return bool(self._waiting or self._running)
|
|
|
|
def get_request_state(self, request_id: str) -> DiffusionRequestState | None:
|
|
return self._request_states.get(request_id)
|
|
|
|
def pop_request_state(self, request_id: str) -> DiffusionRequestState | None:
|
|
self._pop_extra_request_state(request_id)
|
|
return self._request_states.pop(request_id, None)
|
|
|
|
def preempt_request(self, request_id: str) -> bool:
|
|
if request_id not in self._request_states:
|
|
return False
|
|
if request_id in self._running:
|
|
self._running.remove(request_id)
|
|
if not self._running:
|
|
self._running_sampling_params_key = None
|
|
self._waiting.appendleft(request_id)
|
|
self._request_states[request_id].status = DiffusionRequestStatus.PREEMPTED
|
|
return True
|
|
return False
|
|
|
|
def finish_requests(self, request_ids: str | list[str], status: DiffusionRequestStatus) -> None:
|
|
assert DiffusionRequestStatus.is_finished(status)
|
|
if isinstance(request_ids, str):
|
|
request_ids = [request_ids]
|
|
self._finish_requests({request_id: status for request_id in request_ids})
|
|
|
|
def close(self) -> None:
|
|
self._request_states.clear()
|
|
self._waiting.clear()
|
|
self._running.clear()
|
|
self._running_sampling_params_key = None
|
|
self._finished_req_ids.clear()
|
|
self._reset_scheduler_state()
|
|
|
|
def _finish_requests(
|
|
self,
|
|
statuses: dict[str, DiffusionRequestStatus],
|
|
errors: dict[str, str | None] | None = None,
|
|
) -> set[str]:
|
|
if not statuses:
|
|
return set()
|
|
|
|
finished_req_ids: set[str] = set()
|
|
running_to_remove: set[str] = set()
|
|
waiting_to_remove: set[str] = set()
|
|
|
|
for request_id, status in statuses.items():
|
|
assert DiffusionRequestStatus.is_finished(status)
|
|
state = self._request_states.get(request_id)
|
|
if state is None or state.is_finished():
|
|
continue
|
|
|
|
finished_req_ids.add(request_id)
|
|
if request_id in self._running:
|
|
running_to_remove.add(request_id)
|
|
if request_id in self._waiting:
|
|
waiting_to_remove.add(request_id)
|
|
|
|
if running_to_remove:
|
|
self._running = [request_id for request_id in self._running if request_id not in running_to_remove]
|
|
if not self._running:
|
|
self._running_sampling_params_key = None
|
|
if waiting_to_remove:
|
|
self._waiting = deque(request_id for request_id in self._waiting if request_id not in waiting_to_remove)
|
|
|
|
for request_id in finished_req_ids:
|
|
state = self._request_states[request_id]
|
|
status = statuses[request_id]
|
|
state.status = status
|
|
if status == DiffusionRequestStatus.FINISHED_ERROR:
|
|
state.error = None if errors is None else errors.get(request_id)
|
|
else:
|
|
state.error = None
|
|
|
|
self._finished_req_ids |= finished_req_ids
|
|
return finished_req_ids
|
|
|
|
def _finalize_update_from_output(
|
|
self,
|
|
sched_output: DiffusionSchedulerOutput,
|
|
statuses: dict[str, DiffusionRequestStatus],
|
|
errors: dict[str, str | None] | None = None,
|
|
) -> set[str]:
|
|
# A scheduled request may be aborted after schedule() but before
|
|
# update_from_output() processes the runner output. It is already
|
|
# marked finished at that point, but we still need to surface its id
|
|
# in this update so the engine can observe the terminal state.
|
|
finished_req_ids = {
|
|
request_id for request_id in sched_output.scheduled_request_ids if request_id in self._finished_req_ids
|
|
}
|
|
finished_req_ids |= self._finish_requests(statuses, errors)
|
|
return finished_req_ids
|
|
|
|
def _reset_scheduler_state(self) -> None:
|
|
"""Reset subclass-owned state during initialize()/close()."""
|
|
|
|
def _pop_extra_request_state(self, request_id: str) -> None:
|
|
"""Remove subclass-owned per-request state before popping request state."""
|
|
|
|
def _make_request_state(self, request_id: str, request: OmniDiffusionRequest) -> DiffusionRequestState:
|
|
return DiffusionRequestState(
|
|
request_id=request_id,
|
|
req=request,
|
|
sampling_params_key=get_sampling_params_key(request),
|
|
)
|
|
|
|
def _can_schedule_waiting(self, state: DiffusionRequestState) -> bool:
|
|
if not self._running:
|
|
return True
|
|
|
|
current_key = self._current_sampling_params_key()
|
|
return current_key is not None and current_key == state.sampling_params_key
|
|
|
|
def _current_sampling_params_key(self) -> SamplingParamsKey | None:
|
|
if self._running_sampling_params_key is not None or not self._running:
|
|
return self._running_sampling_params_key
|
|
state = self._request_states.get(self._running[0])
|
|
self._running_sampling_params_key = None if state is None else state.sampling_params_key
|
|
return self._running_sampling_params_key
|