[perf]Qwen3-Omni performance optimization (#3164)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-04-28 11:59:26 +08:00
1 parent 1069a144a8
commit 335e2eb9e4
9 files changed
+274 -97

No files matched your search

+157 -66
View File
@@ -80,6 +80,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
self.inputs_embeds = self._make_buffer(self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False)
# Initialize KV cache manager (preserve vllm_config fallback behavior)
self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config)
self._downstream_payload_cache: dict[str, bool] = {}
def _make_buffer(self, *size, dtype, numpy=True):
# Prevent ray from pinning the buffer due to large size
@@ -141,8 +142,62 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
return sampling_metadata
return replace(sampling_metadata, output_token_ids=output_token_ids)
def _request_final_stage_id(self, req_id: str) -> int | None:
info = self.model_intermediate_buffer.get(req_id)
if not isinstance(info, dict):
req_state = self.requests.get(req_id)
info = getattr(req_state, "additional_information_cpu", None)
if not isinstance(info, dict):
return None
val = info.get("omni_final_stage_id")
if val is None:
return None
try:
return int(val)
except (TypeError, ValueError):
return None
def _request_needs_downstream_stage_payload(self, req_id: str) -> bool:
cached = self._downstream_payload_cache.get(req_id)
if cached is not None:
return cached
# Conservative default: keep payload if marker is missing.
final_stage_id = self._request_final_stage_id(req_id)
needs_payload = final_stage_id is None or final_stage_id > 0
self._downstream_payload_cache[req_id] = needs_payload
return needs_payload
def _maybe_prune_downstream_payload_cache(self) -> None:
# Keep cache size bounded under long-lived serving workloads.
if len(self._downstream_payload_cache) <= max(4096, len(self.requests) * 2):
return
self._downstream_payload_cache = {
req_id: needs for req_id, needs in self._downstream_payload_cache.items() if req_id in self.requests
}
def _warmup_single_request_prefill_compile_path(self) -> None:
# Warm up with a single long prefill request to avoid first-token
# compile spikes from wide prefill shapes.
warmup_tokens = min(int(self.max_num_tokens), 3072)
if warmup_tokens <= 1:
return
try:
with torch.inference_mode():
self._dummy_run(
num_tokens=warmup_tokens,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
force_single_prefill_request=True,
allow_microbatching=False,
skip_eplb=True,
)
torch.cuda.synchronize()
logger.info("Warmed single-request prefill compile path with %d tokens", warmup_tokens)
except Exception as e:
logger.warning("Skipped single-request prefill warmup due to: %s", e)
def capture_model(self) -> int:
result = super().capture_model()
self._warmup_single_request_prefill_compile_path()
self._capture_talker_mtp_graphs()
return result
@@ -833,7 +888,21 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
kv_connector_output = self.kv_connector_output
self.kv_connector_output = None
hidden_states_cpu = hidden_states.detach().to("cpu").contiguous()
self._maybe_prune_downstream_payload_cache()
downstream_req_ids = [rid for rid in req_ids_output_copy if self._request_needs_downstream_stage_payload(rid)]
needs_downstream_payload = len(downstream_req_ids) > 0
downstream_req_id_set = set(downstream_req_ids)
hidden_states_cpu = None
req_hidden_states_cpu: dict[str, torch.Tensor] | None = None
if needs_downstream_payload:
num_valid_tokens = min(
int(scheduler_output.total_num_scheduled_tokens),
int(hidden_states.shape[0]),
)
if len(downstream_req_ids) == len(req_ids_output_copy):
hidden_states_cpu = hidden_states[:num_valid_tokens].detach().to("cpu").contiguous()
else:
req_hidden_states_cpu = {}
num_scheduled_tokens_np = getattr(self, "_omni_num_scheduled_tokens_np", None)
if num_scheduled_tokens_np is None:
req_ids = self.input_batch.req_ids
@@ -841,78 +910,96 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
[scheduler_output.num_scheduled_tokens[rid] for rid in req_ids],
dtype=np.int32,
)
query_start_loc_cpu = self.query_start_loc.cpu
# Prior to applying the post-processing func, extract
# the prefix cached hidden states and multimodal states.
if self.omni_prefix_cache is not None:
(
combined_hidden_states,
combined_multimodal_outputs,
) = self._maybe_get_combined_prefix_cache_tensors(
pooler_output: list[dict[str, object]] | None = None
if needs_downstream_payload:
# Prior to applying the post-processing func, extract
# the prefix cached hidden states and multimodal states.
if self.omni_prefix_cache is not None:
(
combined_hidden_states,
combined_multimodal_outputs,
) = self._maybe_get_combined_prefix_cache_tensors(
hidden_states,
multimodal_outputs,
scheduler_output.num_scheduled_tokens,
)
# Otherwise we don't have the mm CPU data yet, so we still need to build it
if self.omni_prefix_cache is None:
mm_cpu = build_mm_cpu(flatten_payload(multimodal_outputs))
self._process_additional_information_updates(
hidden_states,
multimodal_outputs,
scheduler_output.num_scheduled_tokens,
)
# Otherwise we don't have the mm CPU data yet, so we still need to build it
if self.omni_prefix_cache is None:
mm_cpu = build_mm_cpu(flatten_payload(multimodal_outputs))
self._process_additional_information_updates(
hidden_states,
multimodal_outputs,
num_scheduled_tokens_np,
scheduler_output,
combined_hidden_states,
combined_multimodal_outputs,
)
pooler_output: list[dict[str, object]] = []
for rid in req_ids_output_copy:
idx = req_id_to_index_output_copy[rid]
start = int(self.query_start_loc.cpu[idx])
sched = int(num_scheduled_tokens_np[idx])
end = start + sched
# If prefix cache is enabled, we have already split everything
# by request and converted the states to CPU tensors
req_hidden_states = self._resolve_req_hidden_states(
hidden_states_cpu,
num_scheduled_tokens_np,
scheduler_output,
combined_hidden_states,
rid,
start,
end,
combined_multimodal_outputs,
req_ids_filter=downstream_req_id_set,
)
payload: dict[str, object] = {"hidden": req_hidden_states}
mm_payload: dict[str, object] = {}
if combined_multimodal_outputs or mm_cpu:
if combined_multimodal_outputs:
# Prefix cache enabled; all items have already been processed
# and split apart for each request as needed, and all tensors
# have already been detached to the CPU. The only exception is
# lists, which we keep as passthrough data for consistent behavior
# in postprocess.
for mm_key in combined_multimodal_outputs.keys():
value = combined_multimodal_outputs[mm_key][rid]
if isinstance(value, list):
mm_payload[mm_key] = value[idx] if idx < len(value) else value[0]
else:
mm_payload[mm_key] = value
if req_hidden_states_cpu is not None and combined_hidden_states is None:
for rid in downstream_req_ids:
idx = req_id_to_index_output_copy[rid]
start = int(query_start_loc_cpu[idx])
sched = int(num_scheduled_tokens_np[idx])
end = start + sched
req_hidden_states_cpu[rid] = hidden_states[start:end].detach().to("cpu").contiguous()
pooler_output = []
for rid in req_ids_output_copy:
if rid not in downstream_req_id_set:
pooler_output.append({})
continue
idx = req_id_to_index_output_copy[rid]
start = int(query_start_loc_cpu[idx])
sched = int(num_scheduled_tokens_np[idx])
end = start + sched
# If prefix cache is enabled, we have already split everything
# by request and converted the states to CPU tensors
if req_hidden_states_cpu is not None and combined_hidden_states is None:
req_hidden_states = req_hidden_states_cpu[rid]
else:
# Prefix cache disabled; we still need to process the data
for mm_key, mm_val in mm_cpu.items():
mm_payload[mm_key] = to_payload_element(
element=mm_val,
idx=idx,
start=start,
end=end,
pass_lists_through=False,
seq_len=seq_len,
)
payload.update(mm_payload)
# Flatten nested dicts to dotted keys so pooling_output
# stays dict[str, torch.Tensor] for msgspec serialization.
pooler_output.append(flatten_payload(payload))
req_hidden_states = self._resolve_req_hidden_states(
hidden_states_cpu,
combined_hidden_states,
rid,
start,
end,
)
payload: dict[str, object] = {"hidden": req_hidden_states}
mm_payload: dict[str, object] = {}
if combined_multimodal_outputs or mm_cpu:
if combined_multimodal_outputs:
# Prefix cache enabled; all items have already been processed
# and split apart for each request as needed, and all tensors
# have already been detached to the CPU. The only exception is
# lists, which we keep as passthrough data for consistent behavior
# in postprocess.
for mm_key in combined_multimodal_outputs.keys():
value = combined_multimodal_outputs[mm_key][rid]
if isinstance(value, list):
mm_payload[mm_key] = value[idx] if idx < len(value) else value[0]
else:
mm_payload[mm_key] = value
else:
# Prefix cache disabled; we still need to process the data
for mm_key, mm_val in mm_cpu.items():
mm_payload[mm_key] = to_payload_element(
element=mm_val,
idx=idx,
start=start,
end=end,
pass_lists_through=False,
seq_len=seq_len,
)
payload.update(mm_payload)
# Flatten nested dicts to dotted keys so pooling_output
# stays dict[str, torch.Tensor] for msgspec serialization.
pooler_output.append(flatten_payload(payload))
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
@@ -926,7 +1013,11 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
sampled_token_ids=valid_sampled_token_ids,
logprobs=logprobs_lists,
prompt_logprobs_dict=prompt_logprobs_dict,
pooler_output=(pooler_output if self.vllm_config.model_config.engine_output_type != "text" else None),
pooler_output=(
pooler_output
if (self.vllm_config.model_config.engine_output_type != "text" and needs_downstream_payload)
else None
),
kv_connector_output=kv_connector_output,
ec_connector_output=ec_connector_output if self.supports_mm_inputs else None,
num_nans_in_logits=num_nans_in_logits,