Fix yield token metrics and opt metrics record stats (#1292)
This commit is contained in:
1 parent
830d261f9d
commit
82fd2c3637
6 files changed
+162
-39
No files matched your search
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Unit tests for OmniChatCompletionResponse/StreamResponse metrics field."""
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_omni_chat_completion_response_metrics():
|
||||
"""Test OmniChatCompletionResponse metrics field works correctly."""
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.chat_completion import (
|
||||
OmniChatCompletionResponse,
|
||||
)
|
||||
|
||||
usage = UsageInfo(prompt_tokens=0, completion_tokens=0, total_tokens=0)
|
||||
|
||||
# Default is None
|
||||
response = OmniChatCompletionResponse(id="test-id", created=1234567890, model="test-model", choices=[], usage=usage)
|
||||
assert response.metrics is None
|
||||
|
||||
# Can set metrics and serialize
|
||||
test_metrics = {"thinker_ttft": 0.123, "talker_ttft": 0.456}
|
||||
response = OmniChatCompletionResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="test-model",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
metrics=test_metrics,
|
||||
)
|
||||
assert response.metrics == test_metrics
|
||||
assert "thinker_ttft" in response.model_dump_json()
|
||||
|
||||
|
||||
def test_omni_chat_completion_stream_response_metrics():
|
||||
"""Test OmniChatCompletionStreamResponse metrics and modality fields."""
|
||||
from vllm_omni.entrypoints.openai.protocol.chat_completion import (
|
||||
OmniChatCompletionStreamResponse,
|
||||
)
|
||||
|
||||
response = OmniChatCompletionStreamResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="test-model",
|
||||
choices=[],
|
||||
modality="audio",
|
||||
metrics={"stage_latency": 0.5},
|
||||
)
|
||||
assert response.modality == "audio"
|
||||
assert response.metrics == {"stage_latency": 0.5}
|
||||
@@ -32,7 +32,7 @@ from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams
|
||||
|
||||
# Internal imports (our code)
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.metrics import OrchestratorAggregator, StageRequestStats
|
||||
from vllm_omni.metrics import OrchestratorAggregator
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -402,9 +402,6 @@ class AsyncOmni(OmniBase):
|
||||
all_stages_finished[stage_id] = finished
|
||||
|
||||
if output_to_yield:
|
||||
metrics.record_audio_generated_frames(
|
||||
output_to_yield, engine_outputs.finished, stage_id, request_id
|
||||
)
|
||||
yield output_to_yield
|
||||
|
||||
async def _process_sequential_results(
|
||||
@@ -428,9 +425,6 @@ class AsyncOmni(OmniBase):
|
||||
metrics,
|
||||
)
|
||||
if output_to_yield:
|
||||
metrics.record_audio_generated_frames(
|
||||
output_to_yield, engine_outputs.finished, stage_id, request_id
|
||||
)
|
||||
yield output_to_yield
|
||||
if not isinstance(engine_outputs, list):
|
||||
engine_outputs = [engine_outputs]
|
||||
@@ -504,29 +498,6 @@ class AsyncOmni(OmniBase):
|
||||
|
||||
finished = engine_outputs.finished
|
||||
|
||||
# Mark last output time
|
||||
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
|
||||
|
||||
try:
|
||||
_m: StageRequestStats = result.get("metrics")
|
||||
if _m is not None:
|
||||
# Accumulate generation time
|
||||
metrics.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms
|
||||
|
||||
# For diffusion stages, we also accumulate diffusion time
|
||||
metrics.accumulate_diffusion_metrics(stage.stage_type, req_id, engine_outputs)
|
||||
|
||||
if finished:
|
||||
metrics.on_stage_metrics(stage_id, req_id, _m, stage.final_output_type)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
f"[{self._name}] Failed to process metrics for stage {stage_id}, req {req_id}: {e}",
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
|
||||
)
|
||||
|
||||
output_to_yield = None
|
||||
|
||||
if getattr(stage, "final_output", False):
|
||||
@@ -551,6 +522,23 @@ class AsyncOmni(OmniBase):
|
||||
final_output_type=stage.final_output_type,
|
||||
request_output=engine_outputs,
|
||||
)
|
||||
# Mark last output time
|
||||
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
|
||||
|
||||
metrics.process_stage_metrics(
|
||||
result=result,
|
||||
stage_type=stage.stage_type,
|
||||
stage_id=stage_id,
|
||||
req_id=req_id,
|
||||
engine_outputs=engine_outputs,
|
||||
finished=finished,
|
||||
final_output_type=stage.final_output_type,
|
||||
output_to_yield=output_to_yield,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
|
||||
)
|
||||
|
||||
return engine_outputs, finished, output_to_yield
|
||||
|
||||
|
||||
@@ -821,7 +821,7 @@ class Omni(OmniBase):
|
||||
request_output=engine_outputs,
|
||||
)
|
||||
|
||||
# Record audio generated frames with unified signature
|
||||
# Record audio generated frames (only when finished)
|
||||
try:
|
||||
finished = (
|
||||
engine_outputs.finished
|
||||
@@ -834,7 +834,8 @@ class Omni(OmniBase):
|
||||
else False
|
||||
)
|
||||
)
|
||||
metrics.record_audio_generated_frames(output_to_yield, finished, stage_id, req_id)
|
||||
if finished:
|
||||
metrics.record_audio_generated_frames(output_to_yield, stage_id, req_id)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
f"[{self._name}] Failed to record audio metrics for req {req_id} at stage {stage_id}: {e}",
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionStreamResponse
|
||||
from typing import Any
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse, ChatCompletionStreamResponse
|
||||
|
||||
|
||||
class OmniChatCompletionStreamResponse(ChatCompletionStreamResponse):
|
||||
modality: str | None = "text"
|
||||
metrics: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class OmniChatCompletionResponse(ChatCompletionResponse):
|
||||
metrics: dict[str, Any] | None = None
|
||||
@@ -16,6 +16,7 @@ from pydantic import TypeAdapter
|
||||
from vllm.renderers import RendererLike
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||
|
||||
try:
|
||||
@@ -38,7 +39,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
ChatCompletionStreamResponse,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
@@ -679,6 +679,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
stream_options = request.stream_options
|
||||
include_usage, include_continuous_usage = should_include_usage(stream_options, self.enable_force_include_usage)
|
||||
|
||||
last_metrics: dict[str, Any] | None = None
|
||||
try:
|
||||
async for omni_res in result_generator:
|
||||
final_output_type = omni_res.final_output_type
|
||||
@@ -687,6 +688,9 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
logger.warning(f"final output type: {final_output_type} is not needed by the request")
|
||||
continue
|
||||
|
||||
if omni_res.metrics:
|
||||
last_metrics = omni_res.metrics
|
||||
|
||||
if res.prompt_token_ids is not None:
|
||||
num_prompt_tokens = len(res.prompt_token_ids)
|
||||
if res.encoder_prompt_token_ids is not None:
|
||||
@@ -1224,6 +1228,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
choices=[choice_data],
|
||||
model=model_name,
|
||||
modality=final_output_type,
|
||||
metrics=omni_res.metrics,
|
||||
)
|
||||
|
||||
# handle usage stats if requested & if continuous
|
||||
@@ -1273,13 +1278,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
final_usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=num_cached_tokens)
|
||||
|
||||
final_usage_chunk = ChatCompletionStreamResponse(
|
||||
final_usage_chunk = OmniChatCompletionStreamResponse(
|
||||
id=request_id,
|
||||
object=chunk_object_type,
|
||||
created=created_time,
|
||||
choices=[],
|
||||
model=model_name,
|
||||
usage=final_usage,
|
||||
metrics=last_metrics,
|
||||
)
|
||||
final_usage_data = final_usage_chunk.model_dump_json(exclude_unset=True, exclude_none=True)
|
||||
yield f"data: {final_usage_data}\n\n"
|
||||
@@ -1326,7 +1332,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
) -> ErrorResponse | OmniChatCompletionResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
|
||||
@@ -1348,6 +1354,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
prompt_logprobs = None
|
||||
prompt_token_ids = None
|
||||
kv_transfer_params = None
|
||||
response_metrics: dict[str, Any] | None = None
|
||||
|
||||
# Build requested modalities set for filtering
|
||||
requested_modalities = (
|
||||
@@ -1379,9 +1386,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
else:
|
||||
logger.warning(f"Unsupported final output type: {omni_outputs.final_output_type}")
|
||||
continue
|
||||
if omni_outputs.metrics:
|
||||
response_metrics = omni_outputs.metrics
|
||||
choices.extend(choices_data)
|
||||
|
||||
response = ChatCompletionResponse(
|
||||
response = OmniChatCompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
@@ -1390,6 +1399,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
metrics=response_metrics,
|
||||
)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
|
||||
@@ -217,14 +217,12 @@ class OrchestratorAggregator:
|
||||
def record_audio_generated_frames(
|
||||
self,
|
||||
output_to_yield: Any,
|
||||
finished: bool,
|
||||
stage_id: int,
|
||||
request_id: str,
|
||||
) -> None:
|
||||
try:
|
||||
if (
|
||||
output_to_yield.final_output_type == "audio"
|
||||
and finished
|
||||
and (multimodal_output := output_to_yield.multimodal_output.get("audio")) is not None
|
||||
and len(multimodal_output) > 0
|
||||
):
|
||||
@@ -249,6 +247,73 @@ class OrchestratorAggregator:
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def process_stage_metrics(
|
||||
self,
|
||||
*,
|
||||
result: dict[str, Any],
|
||||
stage_type: str,
|
||||
stage_id: int,
|
||||
req_id: str,
|
||||
engine_outputs: Any,
|
||||
finished: bool,
|
||||
final_output_type: str | None,
|
||||
output_to_yield: Any | None,
|
||||
) -> None:
|
||||
"""Process and record stage metrics.
|
||||
|
||||
Args:
|
||||
result: Result dict containing metrics from stage
|
||||
stage_type: Type of the stage (e.g., 'llm', 'diffusion')
|
||||
stage_id: Stage identifier
|
||||
req_id: Request identifier
|
||||
engine_outputs: Engine output object
|
||||
finished: Whether stage processing is finished
|
||||
final_output_type: Type of final output (e.g., 'text', 'audio')
|
||||
output_to_yield: Output object to attach metrics to
|
||||
"""
|
||||
try:
|
||||
_m: StageRequestStats | None = result.get("metrics")
|
||||
|
||||
# 1. Accumulate metrics from stage stats
|
||||
if _m is not None:
|
||||
self.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms
|
||||
self.accumulate_diffusion_metrics(stage_type, req_id, engine_outputs)
|
||||
if finished:
|
||||
self.on_stage_metrics(stage_id, req_id, _m, final_output_type)
|
||||
|
||||
# 2. No output to yield, nothing more to do
|
||||
if output_to_yield is None:
|
||||
return
|
||||
|
||||
# 3. Not finished yet — empty metrics, skip audio recording
|
||||
if not finished:
|
||||
output_to_yield.metrics = {}
|
||||
return
|
||||
|
||||
# 4. Finished with output: assign text metrics if available
|
||||
output_to_yield.metrics = {}
|
||||
stage_event = next(
|
||||
(evt for evt in reversed(self.stage_events.get(req_id, [])) if evt.stage_id == stage_id),
|
||||
None,
|
||||
)
|
||||
if stage_event is not None and stage_event.final_output_type == "text":
|
||||
output_to_yield.metrics = {
|
||||
"num_tokens_in": stage_event.num_tokens_in,
|
||||
"num_tokens_out": stage_event.num_tokens_out,
|
||||
"stage_id": stage_event.stage_id,
|
||||
"final_output_type": stage_event.final_output_type,
|
||||
}
|
||||
|
||||
# 5. Finished: record audio generated frames
|
||||
self.record_audio_generated_frames(output_to_yield, stage_id, req_id)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to process metrics for stage %s, req %s",
|
||||
stage_id,
|
||||
req_id,
|
||||
)
|
||||
|
||||
def _as_stage_request_stats(
|
||||
self,
|
||||
stage_id: int,
|
||||
|
||||
Reference in new issue
Block a user