[Bug fix] fix e2e_total_tokens and e2e_total_time_ms (#648)

Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com>
This commit is contained in:
Junhong Liu authored and GitHub committed 2026-01-08 20:25:04 +08:00
1 parent 8c1259383c
commit bb13467adc
4 files changed
+56 -9

No files matched your search

+44 -1
View File
@@ -5,7 +5,7 @@ from pathlib import Path
import pytest
from vllm.inputs import PromptType
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
SEED = 42
@@ -98,3 +98,46 @@ async def test_abort():
num_generated_tokens, request_id = await task
assert num_generated_tokens == NUM_EXPECTED_TOKENS
await asyncio.sleep(5)
@pytest.mark.asyncio
async def test_build_and_log_summary(monkeypatch):
from vllm_omni.entrypoints.utils import get_final_stage_id_for_e2e
RealCRS = ClientRequestState
capture_metrics = {}
class MockCRS(RealCRS):
def __init__(self, request_id: str):
super().__init__(request_id)
capture_metrics[request_id] = self
monkeypatch.setattr("vllm_omni.entrypoints.async_omni.ClientRequestState", MockCRS)
monkeypatch.setattr("vllm_omni.entrypoints.client_request_state.ClientRequestState", MockCRS)
with ExitStack() as after:
engine = AsyncOmni(model=model, stage_configs_path=stage_config)
after.callback(engine.shutdown)
prompt = "Hello my name is Robert and "
NUM_EXPECTED_TOKENS = 100
NUM_REQUESTS = 5
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
# Create concurrent requests.
tasks: list[asyncio.Task] = []
for idx, request_id in enumerate(request_ids):
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS)))
# Confirm the requests are okay.
for idx, task in enumerate(tasks):
await task
output_modalities = ["text"]
final_stage_id_for_e2e = get_final_stage_id_for_e2e(
output_modalities, engine.output_modalities, engine.stage_list
)
summary = capture_metrics[request_ids[idx]].metrics.build_and_log_summary(final_stage_id_for_e2e)
# Check that total tokens matches sum of stage tokens.
assert summary["e2e_total_tokens"] == sum(stage["tokens"] for stage in summary["stages"])
# Check that total time matches sum of stage times.
assert summary["e2e_total_time_ms"] >= sum(stage["total_time_ms"] for stage in summary["stages"])
+8 -8
View File
@@ -324,8 +324,8 @@ class AsyncOmni(OmniBase):
# Seed stage-0 queue with all requests
logger.debug(f"[{self._name}] Seeding request into stage-0")
req_state = ClientRequestState(request_id)
req_state.metrics = metrics
self.request_states[request_id] = req_state
# Mark first input time for stage-0
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
@@ -360,11 +360,16 @@ class AsyncOmni(OmniBase):
raise RuntimeError(result) # Request Finished due to error
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
stage.set_engine_outputs(engine_outputs)
if isinstance(engine_outputs, list):
engine_outputs = engine_outputs[0]
finished = engine_outputs.finished
# Mark last output time for this stage whenever we receive outputs
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
try:
_m = asdict(result.get("metrics"))
if _m is not None:
if _m is not None and finished:
metrics.on_stage_metrics(stage_id, req_id, _m)
except Exception as e:
logger.exception(
@@ -373,11 +378,6 @@ class AsyncOmni(OmniBase):
logger.debug(
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
)
stage.set_engine_outputs(engine_outputs)
if isinstance(engine_outputs, list):
engine_outputs = engine_outputs[0]
finished = engine_outputs.finished
if getattr(stage, "final_output", False):
logger.debug(
@@ -388,7 +388,7 @@ class AsyncOmni(OmniBase):
# (only once per request at the designated final stage)
try:
rid_key = str(req_id)
if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done:
if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done and finished:
metrics.on_finalize_request(
stage_id,
req_id,
@@ -1,5 +1,7 @@
import asyncio
from vllm_omni.entrypoints.log_utils import OrchestratorMetrics
class ClientRequestState:
"""Tracks the state of an individual request in the orchestrator."""
@@ -8,3 +10,4 @@ class ClientRequestState:
self.request_id = request_id
self.stage_id: int | None = None
self.queue = queue if queue is not None else asyncio.Queue()
self.metrics: OrchestratorMetrics | None = None
+1
View File
@@ -177,6 +177,7 @@ def record_stage_metrics(
# Only record num_tokens_in for stage 0 (initial prompt)
if stage_id == 0:
stage_data["num_tokens_in"] = int(metrics.get("num_tokens_in", 0))
stage_total_tokens[stage_id] += int(metrics.get("num_tokens_in", 0))
pr_stages[stage_id] = stage_data
except Exception:
pass