[Perf] Bagel KV-ready early forwarding and time step consistency for /v1/chat/completions (#2398)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
08cb436d4e
commit
d40840b114
7 files changed
+102
-37
No files matched your search
@@ -268,6 +268,9 @@ class Orchestrator:
|
||||
continue
|
||||
idle = False
|
||||
|
||||
# Handle prefill-finished KV-ready signals before finished outputs.
|
||||
await self._handle_kv_ready_raw_outputs(stage_id, raw_outputs)
|
||||
|
||||
# 2) Process raw outputs through the output processor
|
||||
request_outputs = await self._process_stage_outputs(stage_id, raw_outputs)
|
||||
|
||||
@@ -313,25 +316,7 @@ class Orchestrator:
|
||||
# CFG companion handling: companions don't produce user-visible output
|
||||
# and don't forward to the next stage directly.
|
||||
if finished and req_id in self._companion_ids:
|
||||
parent_id = self._companion_to_parent.get(req_id)
|
||||
if parent_id is not None:
|
||||
self._companion_done.setdefault(parent_id, set()).add(req_id)
|
||||
logger.debug(
|
||||
"[Orchestrator] CFG companion %s done (parent=%s)",
|
||||
req_id,
|
||||
parent_id,
|
||||
)
|
||||
# Check if parent is waiting and all companions are done
|
||||
if parent_id in self._deferred_parents and self._all_companions_done(parent_id):
|
||||
deferred = self._deferred_parents.pop(parent_id)
|
||||
parent_state = self.request_states.get(parent_id)
|
||||
if parent_state is not None:
|
||||
await self._forward_to_next_stage(
|
||||
parent_id,
|
||||
deferred["stage_id"],
|
||||
deferred["output"],
|
||||
parent_state,
|
||||
)
|
||||
await self._handle_cfg_companion_ready(req_id)
|
||||
self.request_states.pop(req_id, None)
|
||||
return
|
||||
|
||||
@@ -358,17 +343,17 @@ class Orchestrator:
|
||||
}
|
||||
)
|
||||
|
||||
if finished and stage_id < req_state.final_stage_id and not self.async_chunk:
|
||||
# If this parent has CFG companions, defer forwarding until all done
|
||||
if (
|
||||
finished
|
||||
and stage_id < req_state.final_stage_id
|
||||
and not self.async_chunk
|
||||
and not self._next_stage_already_submitted(stage_id, req_state)
|
||||
):
|
||||
if req_id in self._companion_map and not self._all_companions_done(req_id):
|
||||
self._deferred_parents[req_id] = {
|
||||
"stage_id": stage_id,
|
||||
"output": output,
|
||||
}
|
||||
logger.debug(
|
||||
"[Orchestrator] Parent %s deferred, waiting for CFG companions",
|
||||
req_id,
|
||||
)
|
||||
else:
|
||||
await self._forward_to_next_stage(req_id, stage_id, output, req_state)
|
||||
|
||||
@@ -393,6 +378,56 @@ class Orchestrator:
|
||||
done_set = self._companion_done.get(parent_id, set())
|
||||
return all(cid in done_set for cid in role_map.values())
|
||||
|
||||
def _next_stage_already_submitted(self, stage_id: int, req_state: OrchestratorRequestState) -> bool:
|
||||
return (stage_id + 1) in req_state.stage_submit_ts
|
||||
|
||||
async def _handle_cfg_companion_ready(self, req_id: str) -> None:
|
||||
"""Mark a CFG companion as done; if all companions are done, flush deferred parent."""
|
||||
parent_id = self._companion_to_parent.get(req_id)
|
||||
if parent_id is None:
|
||||
return
|
||||
done_set = self._companion_done.setdefault(parent_id, set())
|
||||
if req_id in done_set:
|
||||
return
|
||||
done_set.add(req_id)
|
||||
if parent_id in self._deferred_parents and self._all_companions_done(parent_id):
|
||||
deferred = self._deferred_parents.pop(parent_id)
|
||||
parent_state = self.request_states.get(parent_id)
|
||||
if parent_state is not None and not self._next_stage_already_submitted(deferred["stage_id"], parent_state):
|
||||
await self._forward_to_next_stage(
|
||||
parent_id,
|
||||
deferred["stage_id"],
|
||||
deferred["output"],
|
||||
parent_state,
|
||||
)
|
||||
|
||||
async def _handle_kv_ready_raw_outputs(self, stage_id: int, raw_outputs: EngineCoreOutputs) -> None:
|
||||
"""Forward split requests once stage-0 KV is ready, not only when decode fully finishes."""
|
||||
if self.async_chunk:
|
||||
return
|
||||
for raw_output in raw_outputs.outputs:
|
||||
kv_params = getattr(raw_output, "kv_transfer_params", None)
|
||||
if not (isinstance(kv_params, dict) and kv_params.get("kv_ready")):
|
||||
continue
|
||||
req_id = raw_output.request_id
|
||||
req_state = self.request_states.get(req_id)
|
||||
if req_state is None:
|
||||
continue
|
||||
if req_id in self._companion_ids:
|
||||
await self._handle_cfg_companion_ready(req_id)
|
||||
continue
|
||||
if stage_id >= req_state.final_stage_id:
|
||||
continue
|
||||
if self._next_stage_already_submitted(stage_id, req_state):
|
||||
continue
|
||||
if req_id in self._companion_map and not self._all_companions_done(req_id):
|
||||
self._deferred_parents[req_id] = {
|
||||
"stage_id": stage_id,
|
||||
"output": raw_output,
|
||||
}
|
||||
else:
|
||||
await self._forward_to_next_stage(req_id, stage_id, raw_output, req_state)
|
||||
|
||||
def _build_stage_metrics(
|
||||
self,
|
||||
stage_id: int,
|
||||
|
||||
Reference in new issue
Block a user