From bb13467adc16d911b540b5a42a7025824027acc9 Mon Sep 17 00:00:00 2001 From: Junhong Liu Date: Thu, 8 Jan 2026 20:25:04 +0800 Subject: [PATCH] [Bug fix] fix e2e_total_tokens and e2e_total_time_ms (#648) Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com> --- tests/e2e/online_serving/test_async_omni.py | 45 ++++++++++++++++++- vllm_omni/entrypoints/async_omni.py | 16 +++---- vllm_omni/entrypoints/client_request_state.py | 3 ++ vllm_omni/entrypoints/log_utils.py | 1 + 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/tests/e2e/online_serving/test_async_omni.py b/tests/e2e/online_serving/test_async_omni.py index 83f4c4d1c..a572092b5 100644 --- a/tests/e2e/online_serving/test_async_omni.py +++ b/tests/e2e/online_serving/test_async_omni.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest from vllm.inputs import PromptType -from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState SEED = 42 @@ -98,3 +98,46 @@ async def test_abort(): num_generated_tokens, request_id = await task assert num_generated_tokens == NUM_EXPECTED_TOKENS await asyncio.sleep(5) + + +@pytest.mark.asyncio +async def test_build_and_log_summary(monkeypatch): + from vllm_omni.entrypoints.utils import get_final_stage_id_for_e2e + + RealCRS = ClientRequestState + capture_metrics = {} + + class MockCRS(RealCRS): + def __init__(self, request_id: str): + super().__init__(request_id) + capture_metrics[request_id] = self + + monkeypatch.setattr("vllm_omni.entrypoints.async_omni.ClientRequestState", MockCRS) + monkeypatch.setattr("vllm_omni.entrypoints.client_request_state.ClientRequestState", MockCRS) + + with ExitStack() as after: + engine = AsyncOmni(model=model, stage_configs_path=stage_config) + after.callback(engine.shutdown) + prompt = "Hello my name is Robert and " + NUM_EXPECTED_TOKENS = 100 + NUM_REQUESTS = 5 + request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)] + + # Create concurrent requests. + tasks: list[asyncio.Task] = [] + for idx, request_id in enumerate(request_ids): + tasks.append(asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))) + + # Confirm the requests are okay. + for idx, task in enumerate(tasks): + await task + output_modalities = ["text"] + final_stage_id_for_e2e = get_final_stage_id_for_e2e( + output_modalities, engine.output_modalities, engine.stage_list + ) + summary = capture_metrics[request_ids[idx]].metrics.build_and_log_summary(final_stage_id_for_e2e) + + # Check that total tokens matches sum of stage tokens. + assert summary["e2e_total_tokens"] == sum(stage["tokens"] for stage in summary["stages"]) + # Check that total time matches sum of stage times. + assert summary["e2e_total_time_ms"] >= sum(stage["total_time_ms"] for stage in summary["stages"]) diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index bb496198a..2b3e7abad 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -324,8 +324,8 @@ class AsyncOmni(OmniBase): # Seed stage-0 queue with all requests logger.debug(f"[{self._name}] Seeding request into stage-0") req_state = ClientRequestState(request_id) + req_state.metrics = metrics self.request_states[request_id] = req_state - # Mark first input time for stage-0 metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time() @@ -360,11 +360,16 @@ class AsyncOmni(OmniBase): raise RuntimeError(result) # Request Finished due to error engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm") + stage.set_engine_outputs(engine_outputs) + if isinstance(engine_outputs, list): + engine_outputs = engine_outputs[0] + finished = engine_outputs.finished + # Mark last output time for this stage whenever we receive outputs metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time()) try: _m = asdict(result.get("metrics")) - if _m is not None: + if _m is not None and finished: metrics.on_stage_metrics(stage_id, req_id, _m) except Exception as e: logger.exception( @@ -373,11 +378,6 @@ class AsyncOmni(OmniBase): logger.debug( f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing", ) - stage.set_engine_outputs(engine_outputs) - - if isinstance(engine_outputs, list): - engine_outputs = engine_outputs[0] - finished = engine_outputs.finished if getattr(stage, "final_output", False): logger.debug( @@ -388,7 +388,7 @@ class AsyncOmni(OmniBase): # (only once per request at the designated final stage) try: rid_key = str(req_id) - if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done: + if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done and finished: metrics.on_finalize_request( stage_id, req_id, diff --git a/vllm_omni/entrypoints/client_request_state.py b/vllm_omni/entrypoints/client_request_state.py index f818790dd..3e68abb17 100644 --- a/vllm_omni/entrypoints/client_request_state.py +++ b/vllm_omni/entrypoints/client_request_state.py @@ -1,5 +1,7 @@ import asyncio +from vllm_omni.entrypoints.log_utils import OrchestratorMetrics + class ClientRequestState: """Tracks the state of an individual request in the orchestrator.""" @@ -8,3 +10,4 @@ class ClientRequestState: self.request_id = request_id self.stage_id: int | None = None self.queue = queue if queue is not None else asyncio.Queue() + self.metrics: OrchestratorMetrics | None = None diff --git a/vllm_omni/entrypoints/log_utils.py b/vllm_omni/entrypoints/log_utils.py index ba147f12c..2031038da 100644 --- a/vllm_omni/entrypoints/log_utils.py +++ b/vllm_omni/entrypoints/log_utils.py @@ -177,6 +177,7 @@ def record_stage_metrics( # Only record num_tokens_in for stage 0 (initial prompt) if stage_id == 0: stage_data["num_tokens_in"] = int(metrics.get("num_tokens_in", 0)) + stage_total_tokens[stage_id] += int(metrics.get("num_tokens_in", 0)) pr_stages[stage_id] = stage_data except Exception: pass