Merge main into dev/vllm-align for vLLM 0.31 release
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
008fa25263
252 files changed
+27589
-1237
No files matched your search
@@ -888,6 +888,44 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
merged[req_id] = data
|
||||
return merged
|
||||
|
||||
def _call_prepare_runner_inputs(
|
||||
self,
|
||||
prepare_runner_inputs: Callable[..., tuple[torch.Tensor, torch.Tensor]],
|
||||
*,
|
||||
req_ids: list[str],
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
num_computed_tokens: np.ndarray,
|
||||
num_scheduled_tokens: np.ndarray,
|
||||
input_ids_buffer: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Keep opt-in request metadata outside captured model forwards."""
|
||||
if getattr(self.model, "accepts_runner_sampling_extra_args", False):
|
||||
return prepare_runner_inputs(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
req_ids=req_ids,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
input_ids_buffer=input_ids_buffer,
|
||||
sampling_extra_args=[
|
||||
params.extra_args if params and params.extra_args else {}
|
||||
for params in (self.requests[rid].sampling_params for rid in req_ids)
|
||||
],
|
||||
discard_mask=self.discard_request_mask.np[: len(req_ids)],
|
||||
)
|
||||
return prepare_runner_inputs(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
req_ids=req_ids,
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
input_ids_buffer=input_ids_buffer,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -1137,11 +1175,12 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
# for multimodal position detection, fix decode position offsets).
|
||||
prepare_runner_inputs = getattr(self.model, "prepare_runner_inputs", None)
|
||||
if callable(prepare_runner_inputs):
|
||||
input_ids, positions = prepare_runner_inputs(
|
||||
input_ids, positions = self._call_prepare_runner_inputs(
|
||||
prepare_runner_inputs,
|
||||
req_ids=req_ids[:num_reqs],
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
req_ids=req_ids[:num_reqs],
|
||||
num_computed_tokens=self.input_batch.num_computed_tokens_cpu[:num_reqs],
|
||||
num_scheduled_tokens=num_scheduled_tokens_np[:num_reqs],
|
||||
input_ids_buffer=self.input_ids.gpu[:num_tokens_padded],
|
||||
|
||||
Reference in new issue
Block a user