Fix yield token metrics and opt metrics record stats (#1292)

This commit is contained in:
Junhong Liu authored and GitHub committed 2026-02-12 00:52:05 +08:00
1 parent 830d261f9d
commit 82fd2c3637
6 files changed
+162 -39

No files matched your search

@@ -0,0 +1,52 @@
# SPDX-License-Identifier: Apache-2.0
"""Unit tests for OmniChatCompletionResponse/StreamResponse metrics field."""
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_omni_chat_completion_response_metrics():
"""Test OmniChatCompletionResponse metrics field works correctly."""
from vllm.entrypoints.openai.engine.protocol import UsageInfo
from vllm_omni.entrypoints.openai.protocol.chat_completion import (
OmniChatCompletionResponse,
)
usage = UsageInfo(prompt_tokens=0, completion_tokens=0, total_tokens=0)
# Default is None
response = OmniChatCompletionResponse(id="test-id", created=1234567890, model="test-model", choices=[], usage=usage)
assert response.metrics is None
# Can set metrics and serialize
test_metrics = {"thinker_ttft": 0.123, "talker_ttft": 0.456}
response = OmniChatCompletionResponse(
id="test-id",
created=1234567890,
model="test-model",
choices=[],
usage=usage,
metrics=test_metrics,
)
assert response.metrics == test_metrics
assert "thinker_ttft" in response.model_dump_json()
def test_omni_chat_completion_stream_response_metrics():
"""Test OmniChatCompletionStreamResponse metrics and modality fields."""
from vllm_omni.entrypoints.openai.protocol.chat_completion import (
OmniChatCompletionStreamResponse,
)
response = OmniChatCompletionStreamResponse(
id="test-id",
created=1234567890,
model="test-model",
choices=[],
modality="audio",
metrics={"stage_latency": 0.5},
)
assert response.modality == "audio"
assert response.metrics == {"stage_latency": 0.5}
+18 -30
View File
@@ -32,7 +32,7 @@ from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams
# Internal imports (our code)
from vllm_omni.lora.request import LoRARequest
from vllm_omni.metrics import OrchestratorAggregator, StageRequestStats
from vllm_omni.metrics import OrchestratorAggregator
from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
@@ -402,9 +402,6 @@ class AsyncOmni(OmniBase):
all_stages_finished[stage_id] = finished
if output_to_yield:
metrics.record_audio_generated_frames(
output_to_yield, engine_outputs.finished, stage_id, request_id
)
yield output_to_yield
async def _process_sequential_results(
@@ -428,9 +425,6 @@ class AsyncOmni(OmniBase):
metrics,
)
if output_to_yield:
metrics.record_audio_generated_frames(
output_to_yield, engine_outputs.finished, stage_id, request_id
)
yield output_to_yield
if not isinstance(engine_outputs, list):
engine_outputs = [engine_outputs]
@@ -504,29 +498,6 @@ class AsyncOmni(OmniBase):
finished = engine_outputs.finished
# Mark last output time
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
try:
_m: StageRequestStats = result.get("metrics")
if _m is not None:
# Accumulate generation time
metrics.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms
# For diffusion stages, we also accumulate diffusion time
metrics.accumulate_diffusion_metrics(stage.stage_type, req_id, engine_outputs)
if finished:
metrics.on_stage_metrics(stage_id, req_id, _m, stage.final_output_type)
except Exception as e:
logger.exception(
f"[{self._name}] Failed to process metrics for stage {stage_id}, req {req_id}: {e}",
)
logger.debug(
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
)
output_to_yield = None
if getattr(stage, "final_output", False):
@@ -551,6 +522,23 @@ class AsyncOmni(OmniBase):
final_output_type=stage.final_output_type,
request_output=engine_outputs,
)
# Mark last output time
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
metrics.process_stage_metrics(
result=result,
stage_type=stage.stage_type,
stage_id=stage_id,
req_id=req_id,
engine_outputs=engine_outputs,
finished=finished,
final_output_type=stage.final_output_type,
output_to_yield=output_to_yield,
)
logger.debug(
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
)
return engine_outputs, finished, output_to_yield
+3 -2
View File
@@ -821,7 +821,7 @@ class Omni(OmniBase):
request_output=engine_outputs,
)
# Record audio generated frames with unified signature
# Record audio generated frames (only when finished)
try:
finished = (
engine_outputs.finished
@@ -834,7 +834,8 @@ class Omni(OmniBase):
else False
)
)
metrics.record_audio_generated_frames(output_to_yield, finished, stage_id, req_id)
if finished:
metrics.record_audio_generated_frames(output_to_yield, stage_id, req_id)
except Exception as e:
logger.exception(
f"[{self._name}] Failed to record audio metrics for req {req_id} at stage {stage_id}: {e}",
@@ -1,5 +1,12 @@
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionStreamResponse
from typing import Any
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse, ChatCompletionStreamResponse
class OmniChatCompletionStreamResponse(ChatCompletionStreamResponse):
modality: str | None = "text"
metrics: dict[str, Any] | None = None
class OmniChatCompletionResponse(ChatCompletionResponse):
metrics: dict[str, Any] | None = None
+14 -4
View File
@@ -16,6 +16,7 @@ from pydantic import TypeAdapter
from vllm.renderers import RendererLike
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
try:
@@ -38,7 +39,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
ChatCompletionStreamResponse,
ChatMessage,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
@@ -679,6 +679,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
stream_options = request.stream_options
include_usage, include_continuous_usage = should_include_usage(stream_options, self.enable_force_include_usage)
last_metrics: dict[str, Any] | None = None
try:
async for omni_res in result_generator:
final_output_type = omni_res.final_output_type
@@ -687,6 +688,9 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
logger.warning(f"final output type: {final_output_type} is not needed by the request")
continue
if omni_res.metrics:
last_metrics = omni_res.metrics
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if res.encoder_prompt_token_ids is not None:
@@ -1224,6 +1228,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
choices=[choice_data],
model=model_name,
modality=final_output_type,
metrics=omni_res.metrics,
)
# handle usage stats if requested & if continuous
@@ -1273,13 +1278,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
if self.enable_prompt_tokens_details and num_cached_tokens:
final_usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=num_cached_tokens)
final_usage_chunk = ChatCompletionStreamResponse(
final_usage_chunk = OmniChatCompletionStreamResponse(
id=request_id,
object=chunk_object_type,
created=created_time,
choices=[],
model=model_name,
usage=final_usage,
metrics=last_metrics,
)
final_usage_data = final_usage_chunk.model_dump_json(exclude_unset=True, exclude_none=True)
yield f"data: {final_usage_data}\n\n"
@@ -1326,7 +1332,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
) -> ErrorResponse | ChatCompletionResponse:
) -> ErrorResponse | OmniChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -1348,6 +1354,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
prompt_logprobs = None
prompt_token_ids = None
kv_transfer_params = None
response_metrics: dict[str, Any] | None = None
# Build requested modalities set for filtering
requested_modalities = (
@@ -1379,9 +1386,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
else:
logger.warning(f"Unsupported final output type: {omni_outputs.final_output_type}")
continue
if omni_outputs.metrics:
response_metrics = omni_outputs.metrics
choices.extend(choices_data)
response = ChatCompletionResponse(
response = OmniChatCompletionResponse(
id=request_id,
created=created_time,
model=model_name,
@@ -1390,6 +1399,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
prompt_logprobs=prompt_logprobs,
prompt_token_ids=prompt_token_ids,
kv_transfer_params=kv_transfer_params,
metrics=response_metrics,
)
# Log complete response if output logging is enabled
+67 -2
View File
@@ -217,14 +217,12 @@ class OrchestratorAggregator:
def record_audio_generated_frames(
self,
output_to_yield: Any,
finished: bool,
stage_id: int,
request_id: str,
) -> None:
try:
if (
output_to_yield.final_output_type == "audio"
and finished
and (multimodal_output := output_to_yield.multimodal_output.get("audio")) is not None
and len(multimodal_output) > 0
):
@@ -249,6 +247,73 @@ class OrchestratorAggregator:
exc_info=True,
)
def process_stage_metrics(
self,
*,
result: dict[str, Any],
stage_type: str,
stage_id: int,
req_id: str,
engine_outputs: Any,
finished: bool,
final_output_type: str | None,
output_to_yield: Any | None,
) -> None:
"""Process and record stage metrics.
Args:
result: Result dict containing metrics from stage
stage_type: Type of the stage (e.g., 'llm', 'diffusion')
stage_id: Stage identifier
req_id: Request identifier
engine_outputs: Engine output object
finished: Whether stage processing is finished
final_output_type: Type of final output (e.g., 'text', 'audio')
output_to_yield: Output object to attach metrics to
"""
try:
_m: StageRequestStats | None = result.get("metrics")
# 1. Accumulate metrics from stage stats
if _m is not None:
self.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms
self.accumulate_diffusion_metrics(stage_type, req_id, engine_outputs)
if finished:
self.on_stage_metrics(stage_id, req_id, _m, final_output_type)
# 2. No output to yield, nothing more to do
if output_to_yield is None:
return
# 3. Not finished yet — empty metrics, skip audio recording
if not finished:
output_to_yield.metrics = {}
return
# 4. Finished with output: assign text metrics if available
output_to_yield.metrics = {}
stage_event = next(
(evt for evt in reversed(self.stage_events.get(req_id, [])) if evt.stage_id == stage_id),
None,
)
if stage_event is not None and stage_event.final_output_type == "text":
output_to_yield.metrics = {
"num_tokens_in": stage_event.num_tokens_in,
"num_tokens_out": stage_event.num_tokens_out,
"stage_id": stage_event.stage_id,
"final_output_type": stage_event.final_output_type,
}
# 5. Finished: record audio generated frames
self.record_audio_generated_frames(output_to_yield, stage_id, req_id)
except Exception:
logger.exception(
"Failed to process metrics for stage %s, req %s",
stage_id,
req_id,
)
def _as_stage_request_stats(
self,
stage_id: int,