[BugFix][Metrics] Fix per-stage time and token stats in metrics API (#4974)
Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
1 parent
8cb79308c6
commit
cd5daaea6a
18 files changed
+618
-78
No files matched your search
@@ -523,7 +523,11 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
upstream_outputs.append(eco)
|
||||
|
||||
# Handle multimodal-only outputs (generation stages) locally.
|
||||
mm_request_outputs = self._process_mm_only_outputs(mm_only_outputs)
|
||||
mm_request_outputs = self._process_mm_only_outputs(
|
||||
mm_only_outputs,
|
||||
engine_core_timestamp=engine_core_timestamp,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
# Delegate text/pooling outputs to upstream.
|
||||
processed = super().process_outputs(
|
||||
@@ -537,6 +541,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
def _process_mm_only_outputs(
|
||||
self,
|
||||
engine_core_outputs: list[EngineCoreOutput],
|
||||
*,
|
||||
engine_core_timestamp: float | None = None,
|
||||
iteration_stats: IterationStats | None = None,
|
||||
) -> list[OmniRequestOutput | PoolingRequestOutput]:
|
||||
"""Handle outputs from generation stages that have no detokenizer.
|
||||
|
||||
@@ -554,7 +561,15 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
stop_reason = eco.stop_reason
|
||||
kv_transfer_params = eco.kv_transfer_params
|
||||
routed_experts = eco.routed_experts
|
||||
req_state.num_cached_tokens = eco.num_cached_tokens
|
||||
self._update_stats_from_output(
|
||||
req_state,
|
||||
eco,
|
||||
engine_core_timestamp,
|
||||
iteration_stats,
|
||||
)
|
||||
prefill_stats = getattr(eco, "prefill_stats", None)
|
||||
if prefill_stats is not None:
|
||||
req_state.num_cached_tokens = prefill_stats.num_cached_tokens
|
||||
req_state.is_prefilling = False
|
||||
|
||||
is_non_final_audio_chunk = (
|
||||
@@ -578,6 +593,11 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
is_segment_finished = bool(getattr(eco, "is_segment_finished", False))
|
||||
if finish_reason is not None and not is_segment_finished and not is_non_final_audio_chunk:
|
||||
self._finish_request(req_state)
|
||||
self._update_stats_from_finished(
|
||||
req_state,
|
||||
finish_reason,
|
||||
iteration_stats,
|
||||
)
|
||||
return request_outputs
|
||||
|
||||
def _update_stats_from_output(
|
||||
|
||||
Reference in new issue
Block a user