[Core] Support Async & Sync AutoRegressive Scheduling (#3306)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
Alex Brooks authored and GitHub committed 2026-05-05 12:09:45 +08:00
1 parent 703e31fc47
commit bb239fa949
6 files changed
+87 -77

No files matched your search

+6 -4
View File
@@ -313,10 +313,12 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
if finished_reqs and hasattr(self.model, "get_kv_transfer_metadata"):
for req_id, data in finished_reqs.items():
try:
req_idx = self.input_batch.req_id_to_index.get(req_id)
num_computed = (
int(self.input_batch.num_computed_tokens_cpu[req_idx]) if req_idx is not None else None
)
# NOTE: seq_len is the same as num_computed_tokens_cpu in current
# async scheduling, since both exclude async placeholders. We use
# seq_len since we control it, just in case upstream async scheduler
# semantics change in the future.
num_computed = data.get("seq_len")
model_meta = self.model.get_kv_transfer_metadata(
req_id,
num_computed_tokens=num_computed,