Merge main and preserve dispatch backlog across monitor windows

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-21 17:13:27 +08:00
commit ea7de7e3de
769 files changed
+73802 -22805

No files matched your search

+160 -49
View File
@@ -38,6 +38,7 @@ from vllm_omni.diffusion.data import is_diffusion_request_started_output
from vllm_omni.distributed.omni_connectors.utils.config import stage_receives_chunks
from vllm_omni.engine import OmniEngineCoreRequest
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.engine.errors import NativeKVHandoffError
from vllm_omni.engine.membership_controller import MembershipController
from vllm_omni.engine.messages import (
AbortRequestMessage,
@@ -199,6 +200,8 @@ class OrchestratorRequestState:
final_stage_id: int = -1
final_output_stage_ids: set[int] = field(default_factory=set)
finished_final_output_stage_ids: set[int] = field(default_factory=set)
finished_stage_ids: set[int] = field(default_factory=set)
pending_final_output: OutputMessage | None = None
# Wall-clock timestamp when the client-facing engine request was accepted.
request_timestamp: float = 0.0
@@ -219,6 +222,7 @@ class OrchestratorRequestState:
session_owned: bool = False
running_counter_registered: bool = False
request_artifact_dirs: set[str] = field(default_factory=set)
native_kv_transfer_id: str | None = None
@dataclass
@@ -770,20 +774,21 @@ class OrchestratorBase:
float(kv_wait_s),
)
req_state = self.request_states.get(getattr(eco, "request_id", None))
if req_state is None or not req_state.streaming.enabled:
if req_state is None:
continue
segment_finished = bool(getattr(eco, "is_segment_finished", False))
raw_mm = self._completion_multimodal_output(eco, None)
req_state.streaming.segments[stage_id] = StreamingSegmentState(
finished=segment_finished,
token_ids=(self._coerce_int_list(getattr(eco, "new_token_ids", None)) if segment_finished else []),
output_metadata=(dict(raw_mm) if segment_finished and isinstance(raw_mm, dict) else {}),
)
req_state.streaming.new_prompt_len_snapshot = getattr(
eco,
"new_prompt_len_snapshot",
None,
)
if req_state.streaming.enabled:
segment_finished = bool(getattr(eco, "is_segment_finished", False))
raw_mm = self._completion_multimodal_output(eco, None)
req_state.streaming.segments[stage_id] = StreamingSegmentState(
finished=segment_finished,
token_ids=(self._coerce_int_list(getattr(eco, "new_token_ids", None)) if segment_finished else []),
output_metadata=(dict(raw_mm) if segment_finished and isinstance(raw_mm, dict) else {}),
)
req_state.streaming.new_prompt_len_snapshot = getattr(
eco,
"new_prompt_len_snapshot",
None,
)
if await self._apply_raw_terminal_stage_finish(stage_id, eco, req_state):
raw_terminal_request_ids.add(req_state.request_id)
await self._report_duplex_session_request_error(stage_id, replica_id, eco, req_state)
@@ -794,6 +799,11 @@ class OrchestratorBase:
iteration_stats=iteration_stats,
)
if self._stat_logger is not None and (raw_outputs.scheduler_stats is not None or iteration_stats is not None):
key = (stage_id, replica_id)
if key not in self._stage_replica_to_engine_idx:
engine_idx = len(self._stage_replica_to_engine_idx)
self._stat_logger.register_replica(engine_idx, str(stage_id), str(replica_id))
self._stage_replica_to_engine_idx[key] = engine_idx
self._stat_logger.record(
raw_outputs.scheduler_stats,
iteration_stats,
@@ -1373,6 +1383,16 @@ class OrchestratorBase:
try:
await dispatch()
return True
except NativeKVHandoffError as e:
await self._fail_request_client_error(
req_id,
stage_id,
str(e),
status_code=HTTPStatus.BAD_GATEWAY.value,
error_type="NativeKVHandoffError",
release_owners=True,
)
return False
except StageUnavailableError as e:
# No specific replica to evict: the stage already has no live
# replica or the chosen slot was evicted. Fail just this request.
@@ -1504,18 +1524,19 @@ class OrchestratorBase:
``is_segment_finished=False``, but vLLM's output processor may remove the
request state before that EngineCoreOutput is processed.
Only update ``finished_final_output_stage_ids`` here. Request cleanup stays
in ``_route_output`` so downstream async-chunk stages can still deliver
outputs after stage-0 session end.
Record raw stage completion here. Client completion is
resolved after this raw batch passes through the output processor, so a
real processed output wins over the swallowed-terminal fallback.
"""
if getattr(eco, "finish_reason", None) is None:
return False
if getattr(eco, "is_segment_finished", False):
return False
req_state.finished_stage_ids.add(stage_id)
final_output_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id}
if stage_id not in final_output_stage_ids:
return False
return True
req_state.finished_final_output_stage_ids.add(stage_id)
return True
@@ -1568,16 +1589,23 @@ class OrchestratorBase:
replica_id: int,
request_ids: set[str],
) -> None:
"""Finish streaming requests whose raw terminal had no processed output."""
"""Finish requests whose raw terminal had no processed output."""
pool = self.stage_pools[stage_id]
if not pool.final_output:
return
for request_id in request_ids:
req_state = self.request_states.get(request_id)
if req_state is None or req_state.session_owned:
continue
pending = req_state.pending_final_output
if pending is not None:
if set(req_state.stage_submit_ts).issubset(req_state.finished_stage_ids):
req_state.pending_final_output = None
await self.output_async_queue.put(pending)
await self._cleanup_request_ids([request_id, *self._cfg_tracker.cleanup_parent(request_id)])
# A real terminal output is pending; do not replace it with
# the swallowed-output fallback before upstream completion.
continue
if not pool.final_output:
continue
final_output_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id}
if not final_output_stage_ids.issubset(req_state.finished_final_output_stage_ids):
continue
@@ -1651,27 +1679,41 @@ class OrchestratorBase:
await self._cleanup_request_ids([req_id])
return
if finished and not segment_finished:
req_state.finished_stage_ids.add(stage_id)
request_finished = False
if finished and self.stage_pools[stage_id].final_output and not segment_finished:
req_state.finished_final_output_stage_ids.add(stage_id)
final_output_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id}
request_finished = final_output_stage_ids.issubset(req_state.finished_final_output_stage_ids)
if req_state.session_owned:
# Session-owned outputs are delivered to their session through
# ``_intercept_stage_output`` below, never to a request queue.
# Session-owned outputs are delivered through the session interceptor.
pass
elif self.stage_pools[stage_id].final_output:
await self.output_async_queue.put(
OutputMessage(
request_id=req_id,
stage_id=stage_id,
replica_id=replica_id,
engine_outputs=output,
metrics=stage_metrics,
finished=request_finished,
stage_submit_ts=submit_ts,
)
message = OutputMessage(
request_id=req_id,
stage_id=stage_id,
replica_id=replica_id,
engine_outputs=output,
metrics=stage_metrics,
finished=request_finished,
stage_submit_ts=submit_ts,
)
if (
request_finished
and self.async_chunk
and not req_state.streaming.enabled
and not set(req_state.stage_submit_ts).issubset(req_state.finished_stage_ids)
):
# Direct chunk delivery can finish the decoder before the
# upstream engine's terminal output/metrics reaches this loop.
# Keep request state until those messages are routed, otherwise
# the frontend publishes incomplete usage and loses stop reasons.
req_state.pending_final_output = message
request_finished = False
else:
await self.output_async_queue.put(message)
elif stage_metrics is not None:
await self.output_async_queue.put(
StageMetricsMessage(
@@ -1739,6 +1781,12 @@ class OrchestratorBase:
is_final_update=True,
)
pending = req_state.pending_final_output
if pending is not None and set(req_state.stage_submit_ts).issubset(req_state.finished_stage_ids):
req_state.pending_final_output = None
await self.output_async_queue.put(pending)
request_finished = True
if request_finished and not req_state.session_owned:
await self._cleanup_request_ids([req_id, *self._cfg_tracker.cleanup_parent(req_id)])
@@ -2203,12 +2251,13 @@ class OrchestratorBase:
req_id,
req_state,
diffusion_prompt,
submit_kwargs={
"kv_sender_info": self._build_kv_sender_info(
list(getattr(next_client, "engine_input_source", None) or [src_stage_id]),
request_id=req_id,
)
},
submit_kwargs=self._diffusion_submit_kwargs(
req_id,
src_stage_id,
next_client,
req_state,
output,
),
params_override=self._maybe_clone_diffusion_params_for_cfg(req_id, params),
)
self._on_stage_submitted(
@@ -2462,12 +2511,12 @@ class OrchestratorBase:
_t_submit_start = _time.perf_counter()
if next_pool.stage_type == "diffusion":
submit_kwargs = {
"kv_sender_info": self._build_kv_sender_info(
list(getattr(next_pool.stage_client, "engine_input_source", None) or [next_stage_id - 1]),
request_id=request_id,
)
}
submit_kwargs = self._diffusion_submit_kwargs(
request_id,
next_stage_id - 1,
next_pool.stage_client,
req_state,
)
submitted = await self._dispatch_or_fail_request(
lambda: next_pool.submit_initial(
request_id,
@@ -2480,8 +2529,6 @@ class OrchestratorBase:
operation="async-chunk prewarm",
)
else:
import copy
from vllm_omni.distributed.omni_connectors.adapter import compute_talker_prompt_ids_length
try:
@@ -2504,7 +2551,9 @@ class OrchestratorBase:
original_prompt = req_state.prompt
if isinstance(original_prompt, dict):
base_input = copy.deepcopy(original_prompt)
from vllm_omni.engine.request_snapshot import copy_request_snapshot
base_input = copy_request_snapshot(original_prompt)
else:
base_input = {}
@@ -2564,6 +2613,67 @@ class OrchestratorBase:
return True
def _maybe_attach_native_kv_transfer_params(
self,
req_state: OrchestratorRequestState,
prompt: Any,
) -> None:
source_stage_id = 0
if source_stage_id + 1 > req_state.final_stage_id:
return
config = getattr(self.stage_pools[source_stage_id].stage_vllm_config, "kv_transfer_config", None)
if getattr(config, "kv_role", None) != "kv_producer":
return
if self.stage_pools[source_stage_id + 1].stage_type != "diffusion":
return
from vllm_omni.diffusion.diffusion_kv.kv_connector import build_source_kv_transfer_params, mint_transfer_id
transfer_id = mint_transfer_id(req_state.request_id)
req_state.native_kv_transfer_id = transfer_id
params = build_source_kv_transfer_params(
transfer_id=transfer_id,
remote_engine_id=None,
remote_bootstrap_addr=None,
)
for sampling in (req_state.sampling_params_list[source_stage_id], prompt.sampling_params):
sampling.extra_args = {**(sampling.extra_args or {}), "kv_transfer_params": params}
def _diffusion_submit_kwargs(
self,
request_id: str,
source_stage_id: int,
next_client: Any,
req_state: OrchestratorRequestState,
output: Any = None,
) -> dict[str, Any]:
source_stage_ids = list(getattr(next_client, "engine_input_source", None) or [source_stage_id])
if req_state.native_kv_transfer_id is None:
return {"kv_sender_info": self._build_kv_sender_info(source_stage_ids, request_id=request_id)}
if output is None:
raise NativeKVHandoffError("Native KV handoff requires one completed AR source")
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
bootstrap_addr_from_kv_transfer_config,
build_target_kv_transfer_params,
)
source = self.stage_pools[source_stage_id].get_bound_client(request_id)
config = getattr(getattr(source, "vllm_config", None), "kv_transfer_config", None)
if source is None or config is None:
raise NativeKVHandoffError(
f"Native KV handoff for {request_id}: bound AR replica or its KV configuration is unavailable"
)
params = getattr(output, "kv_transfer_params", None)
if not params or "num_transfer_tokens" not in params:
raise NativeKVHandoffError("AR source completed without native KV transfer metadata")
return {
"kv_transfer_params": build_target_kv_transfer_params(
source_params=params,
remote_engine_id=config.engine_id,
remote_bootstrap_addr=bootstrap_addr_from_kv_transfer_config(config),
)
}
def _build_kv_sender_info(
self,
sender_stage_ids: list[int],
@@ -2687,6 +2797,7 @@ class Orchestrator(OrchestratorBase):
request_artifact_dirs=set(msg.request_artifact_dirs or ()),
)
self.request_states[request_id] = req_state
self._maybe_attach_native_kv_transfer_params(req_state, prompt)
self._register_running_request(req_state)
req_state.streaming.enabled = bool(getattr(prompt, "resumable", False))
req_state.stage_submit_ts[stage_id] = _time.time()