[perf] Minicpm o4.5 optimization (#5228)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
acf941da4d
commit
ad3a162eb8
54 files changed
+3861
-3997
No files matched your search
@@ -1361,6 +1361,14 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if nstp is not None and len(nstp) == len(self.input_batch.req_ids):
|
||||
try:
|
||||
model_kwargs_extra["request_token_spans"] = self._compute_request_token_spans(nstp)
|
||||
if getattr(self.model, "requires_request_sample_eligibility", False):
|
||||
model_kwargs_extra["request_sample_eligible"] = [
|
||||
bool(
|
||||
(req := self.requests.get(req_id)) is not None
|
||||
and int(req.num_computed_tokens) + int(nstp[req_index]) >= int(req.num_tokens)
|
||||
)
|
||||
for req_index, req_id in enumerate(self.input_batch.req_ids)
|
||||
]
|
||||
except Exception as e:
|
||||
# Visible on purpose: the fallback is the equal rows-per-request
|
||||
# split, which can re-introduce the cross-request corruption this
|
||||
@@ -1482,14 +1490,17 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
if isinstance(model_buffer, dict) and model_buffer:
|
||||
self._update_intermediate_buffer(new_req.req_id, model_buffer)
|
||||
payload_info = getattr(new_req, "additional_information", None)
|
||||
if isinstance(payload_info, dict):
|
||||
self._update_intermediate_buffer(new_req.req_id, payload_info)
|
||||
decoded_info = deserialize_additional_information(payload_info)
|
||||
if decoded_info:
|
||||
self._update_intermediate_buffer(new_req.req_id, decoded_info)
|
||||
|
||||
if hasattr(scheduler_output.scheduled_cached_reqs, "additional_information"):
|
||||
cached_infos = getattr(scheduler_output.scheduled_cached_reqs, "additional_information", {})
|
||||
if isinstance(cached_infos, dict):
|
||||
for req_id, req_infos in cached_infos.items():
|
||||
self._update_intermediate_buffer(req_id, req_infos)
|
||||
decoded_info = deserialize_additional_information(req_infos)
|
||||
if decoded_info:
|
||||
self._update_intermediate_buffer(req_id, decoded_info)
|
||||
|
||||
def _maybe_attach_mimo_audio_req_infos(
|
||||
self,
|
||||
@@ -1661,9 +1672,8 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
# (e.g., code2wav) so runtime_additional_information can be refreshed
|
||||
# from scheduler cached infos on every step.
|
||||
if hasattr(self.model, "has_preprocess") or hasattr(self.model, "enable_update_additional_information"):
|
||||
if self.vllm_config.model_config.async_chunk:
|
||||
self._update_additional_information(scheduler_output)
|
||||
else:
|
||||
self._update_additional_information(scheduler_output)
|
||||
if not self.vllm_config.model_config.async_chunk:
|
||||
# In full-payload (non-async-chunk) mode, connector-delivered
|
||||
# stage payloads must override any earlier engine-level
|
||||
# additional_information written by the legacy
|
||||
@@ -1672,7 +1682,11 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self._sync_local_stage_payloads()
|
||||
|
||||
if hasattr(self.model, "has_preprocess") and self.model.has_preprocess:
|
||||
preprocess_device = input_ids.device if input_ids is not None else inputs_embeds.device
|
||||
# Multimodal wrappers forward embeddings and therefore intentionally
|
||||
# set ``input_ids`` to None. Model-specific preprocess hooks still
|
||||
# need the scheduled token ids to build or replace those embeddings.
|
||||
preprocess_input_ids = input_ids if input_ids is not None else self.input_ids.gpu[:num_input_tokens]
|
||||
preprocess_device = preprocess_input_ids.device
|
||||
self._maybe_run_batch_preprocess(self.input_batch.req_ids, preprocess_device)
|
||||
|
||||
# Overlay custom prompt_embeds per request for the prompt portion;
|
||||
@@ -1690,21 +1704,27 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
req_ids_b = [item[0] for item in decode_batch_items]
|
||||
start_offsets_b = [item[1] for item in decode_batch_items]
|
||||
req_infos_b = [item[2] for item in decode_batch_items]
|
||||
ids_b = torch.stack([input_ids[offset : offset + 1].reshape(-1)[0] for offset in start_offsets_b])
|
||||
ids_b = torch.stack(
|
||||
[preprocess_input_ids[offset : offset + 1].reshape(-1)[0] for offset in start_offsets_b]
|
||||
)
|
||||
req_input_ids, req_embeds, last_talker_hidden, text_step, updates = batch_decode_preprocess(
|
||||
input_ids=ids_b,
|
||||
req_infos=req_infos_b,
|
||||
)
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = torch.empty(
|
||||
(input_ids.shape[0], req_embeds.shape[-1]),
|
||||
(preprocess_input_ids.shape[0], req_embeds.shape[-1]),
|
||||
device=req_embeds.device,
|
||||
dtype=req_embeds.dtype,
|
||||
)
|
||||
|
||||
offsets_t = torch.tensor(start_offsets_b, device=req_embeds.device, dtype=torch.long)
|
||||
inputs_embeds.index_copy_(0, offsets_t, req_embeds)
|
||||
input_ids.index_copy_(0, offsets_t, req_input_ids.reshape(-1).to(dtype=input_ids.dtype))
|
||||
preprocess_input_ids.index_copy_(
|
||||
0,
|
||||
offsets_t,
|
||||
req_input_ids.reshape(-1).to(dtype=preprocess_input_ids.dtype),
|
||||
)
|
||||
|
||||
dst = slice(len(decode_req_ids), len(decode_req_ids) + len(req_ids_b))
|
||||
self.talker_mtp_input_ids.gpu[dst].copy_(req_input_ids.reshape(-1))
|
||||
@@ -1751,7 +1771,9 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
embed_slice = inputs_embeds[s:e] if inputs_embeds is not None else None
|
||||
req_input_ids, req_embeds, update_dict = self.model.preprocess(
|
||||
input_ids=preprocess_input_ids[s:e], input_embeds=embed_slice, **req_infos
|
||||
input_ids=preprocess_input_ids[s:e],
|
||||
input_embeds=embed_slice,
|
||||
**req_infos,
|
||||
)
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = torch.empty(
|
||||
|
||||
Reference in new issue
Block a user