diff --git a/tests/entrypoints/openai_api/test_serving_chat_metrics.py b/tests/entrypoints/openai_api/test_serving_chat_metrics.py new file mode 100644 index 000000000..d25af6c38 --- /dev/null +++ b/tests/entrypoints/openai_api/test_serving_chat_metrics.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for OmniChatCompletionResponse/StreamResponse metrics field.""" + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_omni_chat_completion_response_metrics(): + """Test OmniChatCompletionResponse metrics field works correctly.""" + from vllm.entrypoints.openai.engine.protocol import UsageInfo + + from vllm_omni.entrypoints.openai.protocol.chat_completion import ( + OmniChatCompletionResponse, + ) + + usage = UsageInfo(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + # Default is None + response = OmniChatCompletionResponse(id="test-id", created=1234567890, model="test-model", choices=[], usage=usage) + assert response.metrics is None + + # Can set metrics and serialize + test_metrics = {"thinker_ttft": 0.123, "talker_ttft": 0.456} + response = OmniChatCompletionResponse( + id="test-id", + created=1234567890, + model="test-model", + choices=[], + usage=usage, + metrics=test_metrics, + ) + assert response.metrics == test_metrics + assert "thinker_ttft" in response.model_dump_json() + + +def test_omni_chat_completion_stream_response_metrics(): + """Test OmniChatCompletionStreamResponse metrics and modality fields.""" + from vllm_omni.entrypoints.openai.protocol.chat_completion import ( + OmniChatCompletionStreamResponse, + ) + + response = OmniChatCompletionStreamResponse( + id="test-id", + created=1234567890, + model="test-model", + choices=[], + modality="audio", + metrics={"stage_latency": 0.5}, + ) + assert response.modality == "audio" + assert response.metrics == {"stage_latency": 0.5} diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 6a3d7393a..ad55c048d 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -32,7 +32,7 @@ from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams # Internal imports (our code) from vllm_omni.lora.request import LoRARequest -from vllm_omni.metrics import OrchestratorAggregator, StageRequestStats +from vllm_omni.metrics import OrchestratorAggregator from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) @@ -402,9 +402,6 @@ class AsyncOmni(OmniBase): all_stages_finished[stage_id] = finished if output_to_yield: - metrics.record_audio_generated_frames( - output_to_yield, engine_outputs.finished, stage_id, request_id - ) yield output_to_yield async def _process_sequential_results( @@ -428,9 +425,6 @@ class AsyncOmni(OmniBase): metrics, ) if output_to_yield: - metrics.record_audio_generated_frames( - output_to_yield, engine_outputs.finished, stage_id, request_id - ) yield output_to_yield if not isinstance(engine_outputs, list): engine_outputs = [engine_outputs] @@ -504,29 +498,6 @@ class AsyncOmni(OmniBase): finished = engine_outputs.finished - # Mark last output time - metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time()) - - try: - _m: StageRequestStats = result.get("metrics") - if _m is not None: - # Accumulate generation time - metrics.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms - - # For diffusion stages, we also accumulate diffusion time - metrics.accumulate_diffusion_metrics(stage.stage_type, req_id, engine_outputs) - - if finished: - metrics.on_stage_metrics(stage_id, req_id, _m, stage.final_output_type) - except Exception as e: - logger.exception( - f"[{self._name}] Failed to process metrics for stage {stage_id}, req {req_id}: {e}", - ) - - logger.debug( - f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing", - ) - output_to_yield = None if getattr(stage, "final_output", False): @@ -551,6 +522,23 @@ class AsyncOmni(OmniBase): final_output_type=stage.final_output_type, request_output=engine_outputs, ) + # Mark last output time + metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time()) + + metrics.process_stage_metrics( + result=result, + stage_type=stage.stage_type, + stage_id=stage_id, + req_id=req_id, + engine_outputs=engine_outputs, + finished=finished, + final_output_type=stage.final_output_type, + output_to_yield=output_to_yield, + ) + + logger.debug( + f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing", + ) return engine_outputs, finished, output_to_yield diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index 711185675..f30cd7d36 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -821,7 +821,7 @@ class Omni(OmniBase): request_output=engine_outputs, ) - # Record audio generated frames with unified signature + # Record audio generated frames (only when finished) try: finished = ( engine_outputs.finished @@ -834,7 +834,8 @@ class Omni(OmniBase): else False ) ) - metrics.record_audio_generated_frames(output_to_yield, finished, stage_id, req_id) + if finished: + metrics.record_audio_generated_frames(output_to_yield, stage_id, req_id) except Exception as e: logger.exception( f"[{self._name}] Failed to record audio metrics for req {req_id} at stage {stage_id}: {e}", diff --git a/vllm_omni/entrypoints/openai/protocol/chat_completion.py b/vllm_omni/entrypoints/openai/protocol/chat_completion.py index cdc93f672..462e53af9 100644 --- a/vllm_omni/entrypoints/openai/protocol/chat_completion.py +++ b/vllm_omni/entrypoints/openai/protocol/chat_completion.py @@ -1,5 +1,12 @@ -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionStreamResponse +from typing import Any + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse, ChatCompletionStreamResponse class OmniChatCompletionStreamResponse(ChatCompletionStreamResponse): modality: str | None = "text" + metrics: dict[str, Any] | None = None + + +class OmniChatCompletionResponse(ChatCompletionResponse): + metrics: dict[str, Any] | None = None diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index c025c5f06..4cc181e09 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -16,6 +16,7 @@ from pydantic import TypeAdapter from vllm.renderers import RendererLike from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt try: @@ -38,7 +39,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, ChatMessage, ) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat @@ -679,6 +679,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): stream_options = request.stream_options include_usage, include_continuous_usage = should_include_usage(stream_options, self.enable_force_include_usage) + last_metrics: dict[str, Any] | None = None try: async for omni_res in result_generator: final_output_type = omni_res.final_output_type @@ -687,6 +688,9 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): logger.warning(f"final output type: {final_output_type} is not needed by the request") continue + if omni_res.metrics: + last_metrics = omni_res.metrics + if res.prompt_token_ids is not None: num_prompt_tokens = len(res.prompt_token_ids) if res.encoder_prompt_token_ids is not None: @@ -1224,6 +1228,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): choices=[choice_data], model=model_name, modality=final_output_type, + metrics=omni_res.metrics, ) # handle usage stats if requested & if continuous @@ -1273,13 +1278,14 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if self.enable_prompt_tokens_details and num_cached_tokens: final_usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=num_cached_tokens) - final_usage_chunk = ChatCompletionStreamResponse( + final_usage_chunk = OmniChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[], model=model_name, usage=final_usage, + metrics=last_metrics, ) final_usage_data = final_usage_chunk.model_dump_json(exclude_unset=True, exclude_none=True) yield f"data: {final_usage_data}\n\n" @@ -1326,7 +1332,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, - ) -> ErrorResponse | ChatCompletionResponse: + ) -> ErrorResponse | OmniChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None @@ -1348,6 +1354,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): prompt_logprobs = None prompt_token_ids = None kv_transfer_params = None + response_metrics: dict[str, Any] | None = None # Build requested modalities set for filtering requested_modalities = ( @@ -1379,9 +1386,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): else: logger.warning(f"Unsupported final output type: {omni_outputs.final_output_type}") continue + if omni_outputs.metrics: + response_metrics = omni_outputs.metrics choices.extend(choices_data) - response = ChatCompletionResponse( + response = OmniChatCompletionResponse( id=request_id, created=created_time, model=model_name, @@ -1390,6 +1399,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): prompt_logprobs=prompt_logprobs, prompt_token_ids=prompt_token_ids, kv_transfer_params=kv_transfer_params, + metrics=response_metrics, ) # Log complete response if output logging is enabled diff --git a/vllm_omni/metrics/stats.py b/vllm_omni/metrics/stats.py index d5c45a0a9..48e394add 100644 --- a/vllm_omni/metrics/stats.py +++ b/vllm_omni/metrics/stats.py @@ -217,14 +217,12 @@ class OrchestratorAggregator: def record_audio_generated_frames( self, output_to_yield: Any, - finished: bool, stage_id: int, request_id: str, ) -> None: try: if ( output_to_yield.final_output_type == "audio" - and finished and (multimodal_output := output_to_yield.multimodal_output.get("audio")) is not None and len(multimodal_output) > 0 ): @@ -249,6 +247,73 @@ class OrchestratorAggregator: exc_info=True, ) + def process_stage_metrics( + self, + *, + result: dict[str, Any], + stage_type: str, + stage_id: int, + req_id: str, + engine_outputs: Any, + finished: bool, + final_output_type: str | None, + output_to_yield: Any | None, + ) -> None: + """Process and record stage metrics. + + Args: + result: Result dict containing metrics from stage + stage_type: Type of the stage (e.g., 'llm', 'diffusion') + stage_id: Stage identifier + req_id: Request identifier + engine_outputs: Engine output object + finished: Whether stage processing is finished + final_output_type: Type of final output (e.g., 'text', 'audio') + output_to_yield: Output object to attach metrics to + """ + try: + _m: StageRequestStats | None = result.get("metrics") + + # 1. Accumulate metrics from stage stats + if _m is not None: + self.accumulated_gen_time_ms[req_id][stage_id] += _m.stage_gen_time_ms + self.accumulate_diffusion_metrics(stage_type, req_id, engine_outputs) + if finished: + self.on_stage_metrics(stage_id, req_id, _m, final_output_type) + + # 2. No output to yield, nothing more to do + if output_to_yield is None: + return + + # 3. Not finished yet — empty metrics, skip audio recording + if not finished: + output_to_yield.metrics = {} + return + + # 4. Finished with output: assign text metrics if available + output_to_yield.metrics = {} + stage_event = next( + (evt for evt in reversed(self.stage_events.get(req_id, [])) if evt.stage_id == stage_id), + None, + ) + if stage_event is not None and stage_event.final_output_type == "text": + output_to_yield.metrics = { + "num_tokens_in": stage_event.num_tokens_in, + "num_tokens_out": stage_event.num_tokens_out, + "stage_id": stage_event.stage_id, + "final_output_type": stage_event.final_output_type, + } + + # 5. Finished: record audio generated frames + self.record_audio_generated_frames(output_to_yield, stage_id, req_id) + + except Exception: + logger.exception( + "Failed to process metrics for stage %s, req %s", + stage_id, + req_id, + ) + def _as_stage_request_stats( self, stage_id: int,