[CI] fix ci and format existing code (#71)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu authored and GitHub committed 2025-11-18 20:54:26 +08:00
1 parent 09fab2ecc9
commit 5707fc78d5
52 files changed
+813 -2155

No files matched your search

+73 -134
View File
@@ -3,12 +3,12 @@ from __future__ import annotations
import logging
import os
import time
from typing import Any, Dict, List, Optional
from typing import Any
from vllm_omni.entrypoints.stage_utils import append_jsonl as _append_jsonl
def remove_old_logs(log_file: Optional[str], num_stages: int) -> None:
def remove_old_logs(log_file: str | None, num_stages: int) -> None:
try:
if not log_file:
return
@@ -48,34 +48,24 @@ def remove_old_logs(log_file: Optional[str], num_stages: int) -> None:
pass
def configure_orchestrator_logger(
logger: logging.Logger, log_file: Optional[str]
) -> None:
def configure_orchestrator_logger(logger: logging.Logger, log_file: str | None) -> None:
try:
if not log_file:
return
has_file_handler = any(
isinstance(h, logging.FileHandler) for h in logger.handlers
)
has_file_handler = any(isinstance(h, logging.FileHandler) for h in logger.handlers)
if not has_file_handler:
fh = logging.FileHandler(log_file)
fh.setLevel(logging.DEBUG)
fh.setFormatter(
logging.Formatter(
"%(asctime)s [PID:%(process)d] %(levelname)s: %(message)s"
)
)
fh.setFormatter(logging.Formatter("%(asctime)s [PID:%(process)d] %(levelname)s: %(message)s"))
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)
except Exception:
pass
def init_stats_paths(
enable_stats: bool, log_file: Optional[str]
) -> tuple[Optional[str], Optional[str]]:
stats_file: Optional[str] = None
overall_file: Optional[str] = None
def init_stats_paths(enable_stats: bool, log_file: str | None) -> tuple[str | None, str | None]:
stats_file: str | None = None
overall_file: str | None = None
try:
if enable_stats and log_file:
stats_file = f"{log_file}.orchestrator.stats.jsonl"
@@ -86,7 +76,7 @@ def init_stats_paths(
return stats_file, overall_file
def _safe_append_jsonl(path: Optional[str], record: Dict[str, Any]) -> None:
def _safe_append_jsonl(path: str | None, record: dict[str, Any]) -> None:
if not path:
return
try:
@@ -96,7 +86,7 @@ def _safe_append_jsonl(path: Optional[str], record: Dict[str, Any]) -> None:
def log_transfer_tx(
stats_file: Optional[str],
stats_file: str | None,
from_stage: int,
to_stage: int,
request_id: Any,
@@ -120,7 +110,7 @@ def log_transfer_tx(
def log_transfer_rx(
stats_file: Optional[str],
stats_file: str | None,
from_stage: int,
to_stage: int,
request_id: Any,
@@ -139,16 +129,14 @@ def log_transfer_rx(
"rx_decode_time_ms": float(rx_decode_time_ms),
"in_flight_time_ms": float(in_flight_time_ms),
"rx_time_per_kb_ms": (
(float(rx_decode_time_ms) / max(float(rx_bytes) / 1024.0, 1e-6))
if rx_bytes > 0
else 0.0
(float(rx_decode_time_ms) / max(float(rx_bytes) / 1024.0, 1e-6)) if rx_bytes > 0 else 0.0
),
},
)
def log_transfer_total(
stats_file: Optional[str],
stats_file: str | None,
from_stage: int,
to_stage: int,
request_id: Any,
@@ -171,16 +159,14 @@ def log_transfer_total(
"rx_decode_time_ms": float(rx_decode_time_ms),
"total_time_ms": float(total_time_ms),
"total_time_per_kb_ms": (
float(total_time_ms) / max(float(size_bytes) / 1024.0, 1e-6)
if size_bytes > 0
else 0.0
float(total_time_ms) / max(float(size_bytes) / 1024.0, 1e-6) if size_bytes > 0 else 0.0
),
},
)
def log_orchestrator_e2e(
stats_file: Optional[str],
stats_file: str | None,
request_id: Any,
final_stage_id: int,
e2e_time_ms: float,
@@ -200,26 +186,20 @@ def log_orchestrator_e2e(
)
def log_orchestrator_summary(
stats_file: Optional[str], summary: Dict[str, Any]
) -> None:
def log_orchestrator_summary(stats_file: str | None, summary: dict[str, Any]) -> None:
_safe_append_jsonl(stats_file, {"type": "orchestrator_summary", **summary})
def log_overall_summary(
overall_stats_file: Optional[str], summary: Dict[str, Any]
) -> None:
def log_overall_summary(overall_stats_file: str | None, summary: dict[str, Any]) -> None:
_safe_append_jsonl(overall_stats_file, {"type": "overall_summary", **summary})
def log_overall_record(
overall_stats_file: Optional[str], record: Dict[str, Any]
) -> None:
def log_overall_record(overall_stats_file: str | None, record: dict[str, Any]) -> None:
_safe_append_jsonl(overall_stats_file, record)
def log_stage_request_stats(
stats_file: Optional[str],
stats_file: str | None,
stage_id: int,
request_id: Any,
batch_size: int,
@@ -248,7 +228,7 @@ def log_stage_request_stats(
def log_stage_running_avg(
stats_file: Optional[str],
stats_file: str | None,
stage_id: int,
total_tokens: int,
total_gen_time_ms: float,
@@ -267,11 +247,11 @@ def log_stage_running_avg(
def log_stage_batch_stats(
stats_file: Optional[str],
stats_file: str | None,
stage_id: int,
batch_size: int,
batch_gen_time_ms: float,
request_ids: List[Any],
request_ids: list[Any],
) -> None:
_safe_append_jsonl(
stats_file,
@@ -286,23 +266,20 @@ def log_stage_batch_stats(
def compute_and_log_stage_request_stats(
stats_file: Optional[str],
stats_file: str | None,
stage_id: int,
request_id: Any,
batch_size: int,
engine_outputs: List[Any],
engine_outputs: list[Any],
stage_gen_time_ms: float,
rx_transfer_bytes: int,
rx_decode_time_ms: float,
) -> None:
"""Compute per-request metrics and log them in one call."""
num_tokens = count_tokens_from_outputs(engine_outputs)
tokens_per_s = (
(num_tokens * 1000.0 / stage_gen_time_ms) if stage_gen_time_ms > 0 else 0.0
)
tokens_per_s = (num_tokens * 1000.0 / stage_gen_time_ms) if stage_gen_time_ms > 0 else 0.0
rx_mbps = (
(float(rx_transfer_bytes) * 8.0)
/ (max(float(rx_decode_time_ms), 1e-6) * 1000.0)
(float(rx_transfer_bytes) * 8.0) / (max(float(rx_decode_time_ms), 1e-6) * 1000.0)
if rx_transfer_bytes > 0
else 0.0
)
@@ -324,21 +301,19 @@ def compute_and_log_stage_request_stats(
def record_stage_metrics(
per_request: Dict[int, Dict[str, Any]],
stage_req_counts: List[int],
stage_total_time_ms: List[float],
stage_total_tokens: List[int],
per_request: dict[int, dict[str, Any]],
stage_req_counts: list[int],
stage_total_time_ms: list[float],
stage_total_tokens: list[int],
stage_id: int,
req_id: Any,
metrics: Dict[str, Any],
metrics: dict[str, Any],
) -> None:
try:
stage_req_counts[stage_id] += 1
stage_total_tokens[stage_id] += int(metrics.get("num_tokens_out", 0))
rid_int = int(req_id)
pr = per_request.setdefault(
rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0}
)
pr = per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
pr_stages = pr["stages"] # type: ignore[index]
pr_stages[stage_id] = {
"stage_gen_time_ms": float(metrics.get("stage_gen_time_ms", 0.0)),
@@ -349,15 +324,15 @@ def record_stage_metrics(
def aggregate_rx_and_maybe_total(
transfer_edge_req: Dict[tuple[int, int, int], Dict[str, float]],
transfer_agg: Dict[tuple[int, int], Dict[str, float]],
per_request: Dict[int, Dict[str, Any]],
transfer_edge_req: dict[tuple[int, int, int], dict[str, float]],
transfer_agg: dict[tuple[int, int], dict[str, float]],
per_request: dict[int, dict[str, Any]],
stage_id: int,
req_id: Any,
rx_bytes: float,
rx_ms: float,
in_flight_ms: float,
) -> Optional[tuple[int, float, float]]:
) -> tuple[int, float, float] | None:
try:
# Update RX aggregates for (stage_id-1 -> stage_id)
if stage_id > 0:
@@ -394,9 +369,7 @@ def aggregate_rx_and_maybe_total(
agg["total_count"] += 1.0
# accumulate per-request transfer totals
try:
pr = per_request.setdefault(
rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0}
)
pr = per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
pr["transfers_ms"] = float(pr.get("transfers_ms", 0.0)) + total_ms # type: ignore[index]
pr["transfers_bytes"] = int(pr.get("transfers_bytes", 0)) + int(rx_bytes) # type: ignore[index]
except Exception:
@@ -408,8 +381,8 @@ def aggregate_rx_and_maybe_total(
def record_sender_transfer_agg(
transfer_agg: Dict[tuple[int, int], Dict[str, float]],
transfer_edge_req: Dict[tuple[int, int, int], Dict[str, float]],
transfer_agg: dict[tuple[int, int], dict[str, float]],
transfer_edge_req: dict[tuple[int, int, int], dict[str, float]],
from_stage: int,
to_stage: int,
req_id: Any,
@@ -447,7 +420,7 @@ def record_sender_transfer_agg(
pass
def count_tokens_from_outputs(engine_outputs: List[Any]) -> int:
def count_tokens_from_outputs(engine_outputs: list[Any]) -> int:
total = 0
for _ro in engine_outputs:
try:
@@ -462,11 +435,11 @@ def count_tokens_from_outputs(engine_outputs: List[Any]) -> int:
def build_stage_summary(
stage_req_counts: List[int],
stage_total_tokens: List[int],
stage_total_time_ms: List[float],
) -> List[Dict[str, Any]]:
summary: List[Dict[str, Any]] = []
stage_req_counts: list[int],
stage_total_tokens: list[int],
stage_total_time_ms: list[float],
) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for sid in range(len(stage_req_counts)):
reqs = stage_req_counts[sid]
tokens = stage_total_tokens[sid]
@@ -487,31 +460,21 @@ def build_stage_summary(
def build_transfer_summary(
transfer_agg: Dict[tuple[int, int], Dict[str, float]]
) -> List[Dict[str, Any]]:
summary: List[Dict[str, Any]] = []
transfer_agg: dict[tuple[int, int], dict[str, float]],
) -> list[dict[str, Any]]:
summary: list[dict[str, Any]] = []
for (src, dst), agg in transfer_agg.items():
sum_bytes = float(agg.get("sum_bytes", 0.0))
sum_ms = float(agg.get("sum_ms", 0.0))
samples = int(agg.get("count", 0.0))
tx_mbps = (
(sum_bytes * 8.0) / (max(sum_ms, 1e-6) * 1000.0) if sum_bytes > 0 else 0.0
)
tx_mbps = (sum_bytes * 8.0) / (max(sum_ms, 1e-6) * 1000.0) if sum_bytes > 0 else 0.0
sum_rx_bytes = float(agg.get("sum_rx_bytes", 0.0))
sum_rx_ms = float(agg.get("sum_rx_ms", 0.0))
samples_rx = int(agg.get("rx_count", 0.0))
rx_mbps = (
(sum_rx_bytes * 8.0) / (max(sum_rx_ms, 1e-6) * 1000.0)
if sum_rx_bytes > 0
else 0.0
)
rx_mbps = (sum_rx_bytes * 8.0) / (max(sum_rx_ms, 1e-6) * 1000.0) if sum_rx_bytes > 0 else 0.0
sum_total_ms = float(agg.get("sum_total_ms", 0.0))
samples_total = int(agg.get("total_count", 0.0))
total_mbps = (
(sum_bytes * 8.0) / (max(sum_total_ms, 1e-6) * 1000.0)
if sum_bytes > 0
else 0.0
)
total_mbps = (sum_bytes * 8.0) / (max(sum_total_ms, 1e-6) * 1000.0) if sum_bytes > 0 else 0.0
summary.append(
{
"from_stage": src,
@@ -537,40 +500,32 @@ class OrchestratorMetrics:
self,
num_stages: int,
enable_stats: bool,
stats_file: Optional[str],
overall_stats_file: Optional[str],
stats_file: str | None,
overall_stats_file: str | None,
wall_start_ts: float,
) -> None:
self.num_stages = int(num_stages)
self.enable_stats = bool(enable_stats)
self.stats_file = stats_file
self.overall_stats_file = overall_stats_file
self.stage_total_time_ms: List[float] = [0.0 for _ in range(self.num_stages)]
self.stage_total_tokens: List[int] = [0 for _ in range(self.num_stages)]
self.stage_req_counts: List[int] = [0 for _ in range(self.num_stages)]
self.transfer_agg: Dict[tuple[int, int], Dict[str, float]] = {}
self.transfer_edge_req: Dict[tuple[int, int, int], Dict[str, float]] = {}
self.stage_total_time_ms: list[float] = [0.0 for _ in range(self.num_stages)]
self.stage_total_tokens: list[int] = [0 for _ in range(self.num_stages)]
self.stage_req_counts: list[int] = [0 for _ in range(self.num_stages)]
self.transfer_agg: dict[tuple[int, int], dict[str, float]] = {}
self.transfer_edge_req: dict[tuple[int, int, int], dict[str, float]] = {}
self.e2e_total_ms: float = 0.0
self.e2e_total_tokens: int = 0
self.e2e_count: int = 0
self.e2e_done: set[int] = set()
self.per_request: Dict[int, Dict[str, Any]] = {}
self.per_request: dict[int, dict[str, Any]] = {}
self.sum_per_request_transfer_ms: float = 0.0
self.wall_start_ts: float = float(wall_start_ts)
self.last_finish_ts: float = float(wall_start_ts)
self.stage_seen_batches: Dict[int, set] = {
sid: set() for sid in range(self.num_stages)
}
self.stage_first_ts: List[Optional[float]] = [
None for _ in range(self.num_stages)
]
self.stage_last_ts: List[Optional[float]] = [
None for _ in range(self.num_stages)
]
self.stage_seen_batches: dict[int, set] = {sid: set() for sid in range(self.num_stages)}
self.stage_first_ts: list[float | None] = [None for _ in range(self.num_stages)]
self.stage_last_ts: list[float | None] = [None for _ in range(self.num_stages)]
def on_stage_metrics(
self, stage_id: int, req_id: Any, metrics: Dict[str, Any]
) -> None:
def on_stage_metrics(self, stage_id: int, req_id: Any, metrics: dict[str, Any]) -> None:
record_stage_metrics(
self.per_request,
self.stage_req_counts,
@@ -585,9 +540,7 @@ class OrchestratorMetrics:
if batch_id_raw is not None:
batch_id = int(batch_id_raw)
if batch_id not in self.stage_seen_batches[stage_id]:
self.stage_total_time_ms[stage_id] += float(
metrics.get("stage_gen_time_ms", 0.0)
)
self.stage_total_time_ms[stage_id] += float(metrics.get("stage_gen_time_ms", 0.0))
self.stage_seen_batches[stage_id].add(batch_id)
except Exception:
pass
@@ -660,9 +613,7 @@ class OrchestratorMetrics:
float(tx_ms),
)
def on_finalize_request(
self, stage_id: int, req_id: Any, engine_outputs: List[Any], req_start_ts: float
) -> None:
def on_finalize_request(self, stage_id: int, req_id: Any, engine_outputs: list[Any], req_start_ts: float) -> None:
try:
rid_int = int(req_id)
except Exception:
@@ -679,9 +630,7 @@ class OrchestratorMetrics:
self.e2e_total_tokens += int(num_tokens)
self.e2e_count += 1
self.e2e_done.add(rid_int)
pr = self.per_request.setdefault(
rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0}
)
pr = self.per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
per_req_record = {
"type": "overall_request",
"request_id": rid_int,
@@ -696,20 +645,16 @@ class OrchestratorMetrics:
log_overall_record(self.overall_stats_file, per_req_record)
if self.enable_stats and self.stats_file:
e2e_tpt = (e2e_ms / num_tokens) if num_tokens > 0 else 0.0
log_orchestrator_e2e(
self.stats_file, req_id, stage_id, e2e_ms, int(num_tokens), e2e_tpt
)
log_orchestrator_e2e(self.stats_file, req_id, stage_id, e2e_ms, int(num_tokens), e2e_tpt)
def build_and_log_summary(self, final_stage_id_for_e2e: int) -> Dict[str, Any]:
def build_and_log_summary(self, final_stage_id_for_e2e: int) -> dict[str, Any]:
# Compute stage summary using wall time between first input and last output per stage
stage_summary: List[Dict[str, Any]] = []
stage_summary: list[dict[str, Any]] = []
for sid in range(self.num_stages):
first_ts = self.stage_first_ts[sid]
last_ts = self.stage_last_ts[sid]
total_ms = (
(max(0.0, (last_ts - first_ts)) * 1000.0)
if (first_ts is not None and last_ts is not None)
else 0.0
(max(0.0, (last_ts - first_ts)) * 1000.0) if (first_ts is not None and last_ts is not None) else 0.0
)
reqs = self.stage_req_counts[sid]
tokens = self.stage_total_tokens[sid]
@@ -726,16 +671,10 @@ class OrchestratorMetrics:
}
)
transfer_summary = build_transfer_summary(self.transfer_agg)
e2e_avg_req = (
(self.e2e_total_ms / self.e2e_count) if self.e2e_count > 0 else 0.0
)
e2e_avg_tok = (
(self.e2e_total_tokens * 1000.0 / self.e2e_total_ms)
if self.e2e_total_ms > 0
else 0.0
)
e2e_avg_req = (self.e2e_total_ms / self.e2e_count) if self.e2e_count > 0 else 0.0
e2e_avg_tok = (self.e2e_total_tokens * 1000.0 / self.e2e_total_ms) if self.e2e_total_ms > 0 else 0.0
wall_time_ms = max(0.0, (self.last_finish_ts - self.wall_start_ts) * 1000.0)
summary: Dict[str, Any] = {
summary: dict[str, Any] = {
"e2e_requests": int(self.e2e_count),
"e2e_total_time_ms": float(wall_time_ms),
"e2e_sum_time_ms": float(self.e2e_total_ms),