[Feature]Support async computation and communication across stages by chunks (#727)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Signed-off-by: Rein Yang <ruiruyang2@gmail.com> Co-authored-by: Rein Yang <ruiruyang2@gmail.com> Co-authored-by: CHEN <116010019@link.cuhk.edu.cn> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
29 files changed
+1258
-396
No files matched your search
@@ -50,6 +50,17 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
- Executes generation process and returns tensors via `pooler_output`.
|
||||
"""
|
||||
|
||||
def _update_request_states(self, scheduler_output: SchedulerOutput):
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
for _, req_id in enumerate(cached_reqs.req_ids):
|
||||
req_state = self.requests.get(req_id)
|
||||
assert req_state is not None
|
||||
req_state.prompt_token_ids = cached_reqs.prompt_token_ids.get(req_id)
|
||||
self.input_batch.remove_request(req_id)
|
||||
# update the request state in self.input_batch
|
||||
self.input_batch.add_request(req_state)
|
||||
self._init_mrope_positions(req_state)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -74,6 +85,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
record_function_or_nullcontext("gpu_model_runner: preprocess"),
|
||||
self.synchronize_input_prep(),
|
||||
):
|
||||
if self.model_config.async_chunk:
|
||||
self._update_request_states(scheduler_output)
|
||||
self._update_states(scheduler_output)
|
||||
if not scheduler_output.total_num_scheduled_tokens:
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
Reference in new issue
Block a user