[Bug fix] fix e2e_total_tokens and e2e_total_time_ms (#648)
Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com>
This commit is contained in:
1 parent
8c1259383c
commit
bb13467adc
4 files changed
+56
-9
No files matched your search
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
|
||||
|
||||
SEED = 42
|
||||
|
||||
@@ -98,3 +98,46 @@ async def test_abort():
|
||||
num_generated_tokens, request_id = await task
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_and_log_summary(monkeypatch):
|
||||
from vllm_omni.entrypoints.utils import get_final_stage_id_for_e2e
|
||||
|
||||
RealCRS = ClientRequestState
|
||||
capture_metrics = {}
|
||||
|
||||
class MockCRS(RealCRS):
|
||||
def __init__(self, request_id: str):
|
||||
super().__init__(request_id)
|
||||
capture_metrics[request_id] = self
|
||||
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.async_omni.ClientRequestState", MockCRS)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.client_request_state.ClientRequestState", MockCRS)
|
||||
|
||||
with ExitStack() as after:
|
||||
engine = AsyncOmni(model=model, stage_configs_path=stage_config)
|
||||
after.callback(engine.shutdown)
|
||||
prompt = "Hello my name is Robert and "
|
||||
NUM_EXPECTED_TOKENS = 100
|
||||
NUM_REQUESTS = 5
|
||||
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
|
||||
|
||||
# Create concurrent requests.
|
||||
tasks: list[asyncio.Task] = []
|
||||
for idx, request_id in enumerate(request_ids):
|
||||
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS)))
|
||||
|
||||
# Confirm the requests are okay.
|
||||
for idx, task in enumerate(tasks):
|
||||
await task
|
||||
output_modalities = ["text"]
|
||||
final_stage_id_for_e2e = get_final_stage_id_for_e2e(
|
||||
output_modalities, engine.output_modalities, engine.stage_list
|
||||
)
|
||||
summary = capture_metrics[request_ids[idx]].metrics.build_and_log_summary(final_stage_id_for_e2e)
|
||||
|
||||
# Check that total tokens matches sum of stage tokens.
|
||||
assert summary["e2e_total_tokens"] == sum(stage["tokens"] for stage in summary["stages"])
|
||||
# Check that total time matches sum of stage times.
|
||||
assert summary["e2e_total_time_ms"] >= sum(stage["total_time_ms"] for stage in summary["stages"])
|
||||
@@ -324,8 +324,8 @@ class AsyncOmni(OmniBase):
|
||||
# Seed stage-0 queue with all requests
|
||||
logger.debug(f"[{self._name}] Seeding request into stage-0")
|
||||
req_state = ClientRequestState(request_id)
|
||||
req_state.metrics = metrics
|
||||
self.request_states[request_id] = req_state
|
||||
|
||||
# Mark first input time for stage-0
|
||||
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
|
||||
|
||||
@@ -360,11 +360,16 @@ class AsyncOmni(OmniBase):
|
||||
raise RuntimeError(result) # Request Finished due to error
|
||||
|
||||
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
|
||||
stage.set_engine_outputs(engine_outputs)
|
||||
if isinstance(engine_outputs, list):
|
||||
engine_outputs = engine_outputs[0]
|
||||
finished = engine_outputs.finished
|
||||
|
||||
# Mark last output time for this stage whenever we receive outputs
|
||||
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
|
||||
try:
|
||||
_m = asdict(result.get("metrics"))
|
||||
if _m is not None:
|
||||
if _m is not None and finished:
|
||||
metrics.on_stage_metrics(stage_id, req_id, _m)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
@@ -373,11 +378,6 @@ class AsyncOmni(OmniBase):
|
||||
logger.debug(
|
||||
f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing",
|
||||
)
|
||||
stage.set_engine_outputs(engine_outputs)
|
||||
|
||||
if isinstance(engine_outputs, list):
|
||||
engine_outputs = engine_outputs[0]
|
||||
finished = engine_outputs.finished
|
||||
|
||||
if getattr(stage, "final_output", False):
|
||||
logger.debug(
|
||||
@@ -388,7 +388,7 @@ class AsyncOmni(OmniBase):
|
||||
# (only once per request at the designated final stage)
|
||||
try:
|
||||
rid_key = str(req_id)
|
||||
if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done:
|
||||
if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done and finished:
|
||||
metrics.on_finalize_request(
|
||||
stage_id,
|
||||
req_id,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
|
||||
from vllm_omni.entrypoints.log_utils import OrchestratorMetrics
|
||||
|
||||
|
||||
class ClientRequestState:
|
||||
"""Tracks the state of an individual request in the orchestrator."""
|
||||
@@ -8,3 +10,4 @@ class ClientRequestState:
|
||||
self.request_id = request_id
|
||||
self.stage_id: int | None = None
|
||||
self.queue = queue if queue is not None else asyncio.Queue()
|
||||
self.metrics: OrchestratorMetrics | None = None
|
||||
@@ -177,6 +177,7 @@ def record_stage_metrics(
|
||||
# Only record num_tokens_in for stage 0 (initial prompt)
|
||||
if stage_id == 0:
|
||||
stage_data["num_tokens_in"] = int(metrics.get("num_tokens_in", 0))
|
||||
stage_total_tokens[stage_id] += int(metrics.get("num_tokens_in", 0))
|
||||
pr_stages[stage_id] = stage_data
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
Reference in new issue
Block a user