[Perf] Bagel KV-ready early forwarding and time step consistency for /v1/chat/completions (#2398)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2026-04-01 08:50:45 +00:00
1 parent 08cb436d4e
commit d40840b114
7 files changed
+102 -37

No files matched your search

+60 -25
View File
@@ -268,6 +268,9 @@ class Orchestrator:
continue
idle = False
# Handle prefill-finished KV-ready signals before finished outputs.
await self._handle_kv_ready_raw_outputs(stage_id, raw_outputs)
# 2) Process raw outputs through the output processor
request_outputs = await self._process_stage_outputs(stage_id, raw_outputs)
@@ -313,25 +316,7 @@ class Orchestrator:
# CFG companion handling: companions don't produce user-visible output
# and don't forward to the next stage directly.
if finished and req_id in self._companion_ids:
parent_id = self._companion_to_parent.get(req_id)
if parent_id is not None:
self._companion_done.setdefault(parent_id, set()).add(req_id)
logger.debug(
"[Orchestrator] CFG companion %s done (parent=%s)",
req_id,
parent_id,
)
# Check if parent is waiting and all companions are done
if parent_id in self._deferred_parents and self._all_companions_done(parent_id):
deferred = self._deferred_parents.pop(parent_id)
parent_state = self.request_states.get(parent_id)
if parent_state is not None:
await self._forward_to_next_stage(
parent_id,
deferred["stage_id"],
deferred["output"],
parent_state,
)
await self._handle_cfg_companion_ready(req_id)
self.request_states.pop(req_id, None)
return
@@ -358,17 +343,17 @@ class Orchestrator:
}
)
if finished and stage_id < req_state.final_stage_id and not self.async_chunk:
# If this parent has CFG companions, defer forwarding until all done
if (
finished
and stage_id < req_state.final_stage_id
and not self.async_chunk
and not self._next_stage_already_submitted(stage_id, req_state)
):
if req_id in self._companion_map and not self._all_companions_done(req_id):
self._deferred_parents[req_id] = {
"stage_id": stage_id,
"output": output,
}
logger.debug(
"[Orchestrator] Parent %s deferred, waiting for CFG companions",
req_id,
)
else:
await self._forward_to_next_stage(req_id, stage_id, output, req_state)
@@ -393,6 +378,56 @@ class Orchestrator:
done_set = self._companion_done.get(parent_id, set())
return all(cid in done_set for cid in role_map.values())
def _next_stage_already_submitted(self, stage_id: int, req_state: OrchestratorRequestState) -> bool:
return (stage_id + 1) in req_state.stage_submit_ts
async def _handle_cfg_companion_ready(self, req_id: str) -> None:
"""Mark a CFG companion as done; if all companions are done, flush deferred parent."""
parent_id = self._companion_to_parent.get(req_id)
if parent_id is None:
return
done_set = self._companion_done.setdefault(parent_id, set())
if req_id in done_set:
return
done_set.add(req_id)
if parent_id in self._deferred_parents and self._all_companions_done(parent_id):
deferred = self._deferred_parents.pop(parent_id)
parent_state = self.request_states.get(parent_id)
if parent_state is not None and not self._next_stage_already_submitted(deferred["stage_id"], parent_state):
await self._forward_to_next_stage(
parent_id,
deferred["stage_id"],
deferred["output"],
parent_state,
)
async def _handle_kv_ready_raw_outputs(self, stage_id: int, raw_outputs: EngineCoreOutputs) -> None:
"""Forward split requests once stage-0 KV is ready, not only when decode fully finishes."""
if self.async_chunk:
return
for raw_output in raw_outputs.outputs:
kv_params = getattr(raw_output, "kv_transfer_params", None)
if not (isinstance(kv_params, dict) and kv_params.get("kv_ready")):
continue
req_id = raw_output.request_id
req_state = self.request_states.get(req_id)
if req_state is None:
continue
if req_id in self._companion_ids:
await self._handle_cfg_companion_ready(req_id)
continue
if stage_id >= req_state.final_stage_id:
continue
if self._next_stage_already_submitted(stage_id, req_state):
continue
if req_id in self._companion_map and not self._all_companions_done(req_id):
self._deferred_parents[req_id] = {
"stage_id": stage_id,
"output": raw_output,
}
else:
await self._forward_to_next_stage(req_id, stage_id, raw_output, req_state)
def _build_stage_metrics(
self,
stage_id: int,