[Feature] Opt metrics structure (#891)
Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com> Signed-off-by: Junhong Liu <ljh_lbj@163.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
ac397ae2a0
commit
9af6fb9a1f
23 files changed
+1151
-728
No files matched your search
@@ -9,7 +9,7 @@ from vllm import SamplingParams
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
@@ -115,54 +115,3 @@ async def test_abort():
|
||||
num_generated_tokens, request_id = await task
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_and_log_summary(monkeypatch):
|
||||
from vllm_omni.entrypoints.utils import get_final_stage_id_for_e2e
|
||||
|
||||
RealCRS = ClientRequestState
|
||||
capture_metrics = {}
|
||||
|
||||
class MockCRS(RealCRS):
|
||||
def __init__(self, request_id: str):
|
||||
super().__init__(request_id)
|
||||
capture_metrics[request_id] = self
|
||||
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.async_omni.ClientRequestState", MockCRS)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.client_request_state.ClientRequestState", MockCRS)
|
||||
|
||||
with ExitStack() as after:
|
||||
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
|
||||
engine = AsyncOmni(
|
||||
model=model,
|
||||
stage_configs_path=stage_config,
|
||||
shm_threshold_bytes=sys.maxsize,
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
prompt = "Hello my name is Robert and "
|
||||
NUM_EXPECTED_TOKENS = 64
|
||||
NUM_REQUESTS = 3
|
||||
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
|
||||
|
||||
# Create concurrent requests.
|
||||
tasks: list[asyncio.Task] = []
|
||||
for idx, request_id in enumerate(request_ids):
|
||||
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS)))
|
||||
|
||||
# Confirm the requests are okay.
|
||||
for idx, task in enumerate(tasks):
|
||||
await task
|
||||
output_modalities = ["text"]
|
||||
final_stage_id_for_e2e = get_final_stage_id_for_e2e(
|
||||
output_modalities, engine.output_modalities, engine.stage_list
|
||||
)
|
||||
summary = capture_metrics[request_ids[idx]].metrics.build_and_log_summary(final_stage_id_for_e2e)
|
||||
|
||||
# Check that total tokens matches sum of stage tokens.
|
||||
assert summary["e2e_total_tokens"] == sum(stage["tokens"] for stage in summary["stages"])
|
||||
# Check that total time matches sum of stage times.
|
||||
assert summary["e2e_total_time_ms"] >= sum(stage["total_time_ms"] for stage in summary["stages"])
|
||||
@@ -298,9 +298,9 @@ def _setup_log_mocks(monkeypatch):
|
||||
# Mock OrchestratorMetrics to be a simple class that doesn't require file operations
|
||||
|
||||
class _FakeOrchestratorMetrics:
|
||||
def __init__(self, num_stages, enable_stats, wall_start_ts):
|
||||
def __init__(self, num_stages, log_stats, wall_start_ts):
|
||||
self.num_stages = num_stages
|
||||
self.enable_stats = enable_stats
|
||||
self.log_stats = log_stats
|
||||
self.stage_first_ts = [None] * num_stages
|
||||
self.stage_last_ts = [None] * num_stages
|
||||
self.e2e_done = set()
|
||||
@@ -318,7 +318,7 @@ def _setup_log_mocks(monkeypatch):
|
||||
return "Fake summary"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.omni.OrchestratorMetrics",
|
||||
"vllm_omni.entrypoints.omni.OrchestratorAggregator",
|
||||
_FakeOrchestratorMetrics,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.metrics import OrchestratorAggregator
|
||||
from vllm_omni.metrics.stats import RequestE2EStats, StageRequestStats, StageStats
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _get_request_entry(table: list[dict], request_id: str) -> dict:
|
||||
for entry in table:
|
||||
if entry.get("request_id") == request_id:
|
||||
return entry
|
||||
raise AssertionError(f"request_id={request_id} not found")
|
||||
|
||||
|
||||
def test_orchestrator_aggregator_builds_summary() -> None:
|
||||
agg = OrchestratorAggregator(num_stages=2, log_stats=True, wall_start_ts=0.0, final_stage_id_for_e2e=1)
|
||||
agg.stage_first_ts[0] = 0.0
|
||||
agg.stage_last_ts[0] = 0.03
|
||||
agg.stage_first_ts[1] = 0.05
|
||||
agg.stage_last_ts[1] = 0.07
|
||||
|
||||
agg.on_forward(0, 1, "r1", size_bytes=1024, tx_ms=5.0, used_shm=False)
|
||||
|
||||
agg.on_stage_metrics(
|
||||
0,
|
||||
"r1",
|
||||
StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=3,
|
||||
num_tokens_out=3,
|
||||
stage_gen_time_ms=30.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(),
|
||||
),
|
||||
)
|
||||
agg.on_stage_metrics(
|
||||
1,
|
||||
"r1",
|
||||
StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=0,
|
||||
num_tokens_out=4,
|
||||
stage_gen_time_ms=20.0,
|
||||
rx_transfer_bytes=1024,
|
||||
rx_decode_time_ms=5.0,
|
||||
rx_in_flight_time_ms=2.0,
|
||||
stage_stats=StageStats(),
|
||||
),
|
||||
)
|
||||
agg.on_finalize_request(1, "r1", req_start_ts=0.0)
|
||||
|
||||
summary = agg.build_and_log_summary()
|
||||
overall = summary["overall_summary"]
|
||||
assert overall["e2e_requests"] == 1
|
||||
|
||||
stage_entry = _get_request_entry(summary["stage_table"], "r1")
|
||||
stage_ids = [row["stage_id"] for row in stage_entry["stages"]]
|
||||
assert stage_ids == [0, 1]
|
||||
|
||||
transfer_entry = _get_request_entry(summary["trans_table"], "r1")
|
||||
assert transfer_entry["transfers"][0]["edge"] == "0->1"
|
||||
assert transfer_entry["transfers"][0]["size_kbytes"] == 1.0
|
||||
|
||||
e2e_entry = _get_request_entry(summary["e2e_table"], "r1")
|
||||
assert e2e_entry["e2e_total_tokens"] == 10
|
||||
|
||||
|
||||
def test_build_and_log_summary_e2e_only() -> None:
|
||||
agg = OrchestratorAggregator(num_stages=1, log_stats=True, wall_start_ts=0.0, final_stage_id_for_e2e=0)
|
||||
agg.e2e_events.append(
|
||||
RequestE2EStats(
|
||||
request_id="r",
|
||||
e2e_total_ms=10.0,
|
||||
e2e_total_tokens=5,
|
||||
transfers_total_time_ms=0.0,
|
||||
transfers_total_bytes=0,
|
||||
)
|
||||
)
|
||||
|
||||
summary = agg.build_and_log_summary()
|
||||
e2e_entry = _get_request_entry(summary["e2e_table"], "r")
|
||||
assert e2e_entry["e2e_total_tokens"] == 5
|
||||
stage_entry = _get_request_entry(summary["stage_table"], "r")
|
||||
assert stage_entry["stages"] == []
|
||||
|
||||
|
||||
def test_build_and_log_summary_multiple_requests() -> None:
|
||||
agg = OrchestratorAggregator(
|
||||
num_stages=2, log_stats=True, wall_start_ts=0.0, final_stage_id_for_e2e={"r1": 1, "r2": 0}
|
||||
)
|
||||
|
||||
# Request r1 goes through both stages
|
||||
agg.on_stage_metrics(
|
||||
0,
|
||||
"r1",
|
||||
StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=2,
|
||||
num_tokens_out=4,
|
||||
stage_gen_time_ms=10.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(),
|
||||
),
|
||||
)
|
||||
agg.on_stage_metrics(
|
||||
1,
|
||||
"r1",
|
||||
StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=4,
|
||||
num_tokens_out=5,
|
||||
stage_gen_time_ms=8.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(),
|
||||
),
|
||||
)
|
||||
agg.on_finalize_request(1, "r1", req_start_ts=0.0)
|
||||
|
||||
# Request r2 only goes through stage 0
|
||||
agg.on_stage_metrics(
|
||||
0,
|
||||
"r2",
|
||||
StageRequestStats(
|
||||
batch_id=2,
|
||||
batch_size=1,
|
||||
num_tokens_in=1,
|
||||
num_tokens_out=2,
|
||||
stage_gen_time_ms=12.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(),
|
||||
),
|
||||
)
|
||||
agg.on_finalize_request(0, "r2", req_start_ts=0.0)
|
||||
|
||||
summary = agg.build_and_log_summary()
|
||||
assert len(summary["stage_table"]) == 2
|
||||
assert {entry["request_id"] for entry in summary["e2e_table"]} == {"r1", "r2"}
|
||||
# Check that r1 has two stages and r2 has one
|
||||
r1_stage_entry = next(e for e in summary["stage_table"] if e["request_id"] == "r1")
|
||||
r2_stage_entry = next(e for e in summary["stage_table"] if e["request_id"] == "r2")
|
||||
assert len(r1_stage_entry["stages"]) == 2
|
||||
assert len(r2_stage_entry["stages"]) == 1
|
||||
Reference in new issue
Block a user