[Feature]Support async computation and communication across stages by chunks (#727)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Signed-off-by: Rein Yang <ruiruyang2@gmail.com>
Co-authored-by: Rein Yang <ruiruyang2@gmail.com>
Co-authored-by: CHEN <116010019@link.cuhk.edu.cn>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-01-26 11:21:24 +08:00
1 parent 578ed1967e
commit 52bf1e2f40
29 files changed
+1258 -396

No files matched your search

@@ -50,6 +50,17 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
- Executes generation process and returns tensors via `pooler_output`.
"""
def _update_request_states(self, scheduler_output: SchedulerOutput):
cached_reqs = scheduler_output.scheduled_cached_reqs
for _, req_id in enumerate(cached_reqs.req_ids):
req_state = self.requests.get(req_id)
assert req_state is not None
req_state.prompt_token_ids = cached_reqs.prompt_token_ids.get(req_id)
self.input_batch.remove_request(req_id)
# update the request state in self.input_batch
self.input_batch.add_request(req_state)
self._init_mrope_positions(req_state)
@torch.inference_mode()
def execute_model(
self,
@@ -74,6 +85,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
record_function_or_nullcontext("gpu_model_runner: preprocess"),
self.synchronize_input_prep(),
):
if self.model_config.async_chunk:
self._update_request_states(scheduler_output)
self._update_states(scheduler_output)
if not scheduler_output.total_num_scheduled_tokens:
return EMPTY_MODEL_RUNNER_OUTPUT