diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 7b535033e..8b21acfa7 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -1,5 +1,5 @@ steps: - - label: "Omni Model Test with H100" + - label: ":full_moon: Omni Model Test with H100" timeout_in_minutes: 90 depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" @@ -41,7 +41,7 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: "Omni Model Test" + - label: ":full_moon: Omni Model Test" timeout_in_minutes: 60 depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" @@ -56,18 +56,22 @@ steps: always-pull: true shm-size: "8gb" propagate-environment: true + shm-size: "8gb" environment: - "HF_HOME=/fsx/hf_cache" volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Omni Model Perf Test" - timeout_in_minutes: 120 + - label: ":full_moon: Omni Model Perf Test with H100" + key: nightly-performance + timeout_in_minutes: 180 depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export BENCHMARK_DIR=tests - pytest -s -v tests/perf/scripts/run_benchmark.py + - buildkite-agent artifact upload "tests/*.json" agents: queue: "mithril-h100-pool" plugins: @@ -96,3 +100,18 @@ steps: hostPath: path: /mnt/hf-cache type: DirectoryOrCreate + + - label: ":email: Nightly Perf Collection & Email" + key: nightly-perf-distribution + depends_on: nightly-performance + if: build.env("NIGHTLY") == "1" + commands: + - pip install openpyxl + - export DEFAULT_INPUT_DIR=tests + - export DEFAULT_OUTPUT_DIR=tests + - buildkite-agent artifact download "tests/*.json" . --step nightly-performance + - python tools/nightly/generate_nightly_perf_excel.py + - python tools/nightly/send_nightly_perf_email.py + - buildkite-agent artifact upload "tests/*.xlsx" + agents: + queue: "cpu_queue_premerge" diff --git a/pyproject.toml b/pyproject.toml index 73ad69c25..95e43d871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ dev = [ "imageio[ffmpeg]>=0.6.0", "opencv-python>=4.12.0.88", "mooncake-transfer-engine==0.3.8.post1", - "av" # for ComfyUI tests + "av", # for ComfyUI tests + "openpyxl>=3.0.0", # for nightly CI ] docs = [ diff --git a/tests/perf/scripts/run_benchmark.py b/tests/perf/scripts/run_benchmark.py index 688666162..113c7e826 100644 --- a/tests/perf/scripts/run_benchmark.py +++ b/tests/perf/scripts/run_benchmark.py @@ -1,9 +1,5 @@ -import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" - import json +import os import subprocess import threading from datetime import datetime @@ -14,6 +10,9 @@ import pytest from tests.conftest import OmniServer, modify_stage_config +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + def load_configs(config_path: str) -> list[dict[str, Any]]: try: @@ -123,6 +122,8 @@ def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_promp "--endpoint", "/v1/chat/completions", "--save-result", + "--result-dir", + os.environ.get("BENCHMARK_DIR", "tests"), "--result-filename", result_filename, ] @@ -137,9 +138,9 @@ def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_promp for line in iter(process.stderr.readline, ""): print(line, end=" ") - if "--result-dir" in args: - index = args.index("--result-dir") - result_dir = args[index + 1] + if "--result-dir" in command: + index = command.index("--result-dir") + result_dir = command[index + 1] else: result_dir = "./" diff --git a/tools/nightly/generate_nightly_perf_excel.py b/tools/nightly/generate_nightly_perf_excel.py new file mode 100644 index 000000000..a9376475d --- /dev/null +++ b/tools/nightly/generate_nightly_perf_excel.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +""" +Generate a nightly Excel performance report from JSON results. + +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +from collections.abc import Iterable, Sequence +from datetime import datetime, timezone +from typing import Any + +from openpyxl import Workbook +from openpyxl.styles import PatternFill +from openpyxl.utils import get_column_letter + +LOGGER = logging.getLogger(__name__) + +GREY_BLOCK_FILL = PatternFill(start_color="D3D3D3", fill_type="solid") + +# Benchmark metric columns: grey the latest row's cell when value changed vs previous date. +BENCHMARK_COLUMNS: tuple[str, ...] = ( + "num_prompts", + "request_rate", + "burstiness", + "max_concurrency", + "duration", + "completed", + "failed", + "request_throughput", + "output_throughput", + "total_token_throughput", + "mean_ttft_ms", + "p99_ttft_ms", + "mean_tpot_ms", + "p99_tpot_ms", + "mean_itl_ms", + "p99_itl_ms", + "mean_e2el_ms", + "p99_e2el_ms", + "mean_audio_rtf", + "p99_audio_rtf", + "mean_audio_duration_s", + "p99_audio_duration_s", +) +# Columns that get float coercion and number format in Excel. Excludes request_rate ("inf" str) +# and max_concurrency (null); leave those as-is. If they become float in the future, they are +# still written correctly from JSON without coercion here. +NUMERIC_FORMAT_COLUMNS: tuple[str, ...] = tuple( + c for c in BENCHMARK_COLUMNS if c not in ("request_rate", "max_concurrency") +) + +_COLUMNS_FILENAME = "nightly_perf_summary_columns.txt" +DEFAULT_INPUT_DIR = os.getenv("DEFAULT_INPUT_DIR") if os.getenv("DEFAULT_INPUT_DIR") else "tests" +DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") if os.getenv("DEFAULT_OUTPUT_DIR") else "tests" + + +def _load_summary_columns(script_dir: str) -> list[str]: + """Load summary column names from a file next to this script; fallback to default if missing.""" + path = os.path.join(script_dir, _COLUMNS_FILENAME) + default = [ + "date", + "endpoint_type", + "backend", + "model_id", + "tokenizer_id", + "num_prompts", + "request_rate", + "burstiness", + "max_concurrency", + "duration", + "completed", + "failed", + "request_throughput", + "output_throughput", + "total_token_throughput", + "mean_ttft_ms", + "p99_ttft_ms", + "mean_tpot_ms", + "p99_tpot_ms", + "mean_itl_ms", + "p99_itl_ms", + "mean_e2el_ms", + "p99_e2el_ms", + "mean_audio_rtf", + "p99_audio_rtf", + "mean_audio_duration_s", + "p99_audio_duration_s", + "commit_sha", + "build_id", + "build_url", + "source_file", + ] + if not os.path.isfile(path): + return default + columns: list[str] = [] + with open(path, encoding="utf-8") as f: + for line in f: + s = line.strip() + if s and not s.startswith("#"): + columns.append(s) + return columns if columns else default + + +def _vllm_omni_root() -> str: + """Resolve vllm-omni repo root: directory that contains a 'tests' subdir (and usually 'tools').""" + path = os.path.dirname(os.path.abspath(__file__)) + while path and path != os.path.dirname(path): + if os.path.isdir(os.path.join(path, "tests")): + return path + path = os.path.dirname(path) + return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + + +def _default_input_dir() -> str: + """Default: vllm-omni root / DEFAULT_INPUT_DIR (where performance JSON files live).""" + root = _vllm_omni_root() + return os.path.join(root, DEFAULT_INPUT_DIR) + + +def _default_output_file() -> str: + """Default: vllm-omni root / DEFAULT_OUTPUT_DIR / nightly_perf_.xlsx.""" + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + return os.path.join(_vllm_omni_root(), DEFAULT_OUTPUT_DIR, f"nightly_perf_{ts}.xlsx") + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Read performance JSON files from vllm-omni/tests/ and generate an Excel report." + ) + parser.add_argument( + "--input-dir", + type=str, + default=_default_input_dir(), + help="Directory containing performance JSON files; default is /DEFAULT_INPUT_DIR.", + ) + parser.add_argument( + "--output-file", + type=str, + default=_default_output_file(), + help="Output path of the Excel report; \ + default is /DEFAULT_OUTPUT_DIR/nightly_perf_.xlsx.", + ) + parser.add_argument( + "--commit-sha", + type=str, + default=None, + help="Optional commit SHA; defaults to environment variable BUILDKITE_COMMIT if unset.", + ) + parser.add_argument( + "--build-id", + type=str, + default=None, + help="Optional build ID; defaults to environment variable BUILDKITE_BUILD_ID if unset.", + ) + parser.add_argument( + "--build-url", + type=str, + default=None, + help="Optional build URL; defaults to environment variable BUILDKITE_BUILD_URL if unset.", + ) + return parser.parse_args() + + +def _load_json_file(path: str) -> dict[str, Any] | None: + """Safely load a single JSON file; return None and log a warning on failure.""" + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as exc: + LOGGER.warning("failed to load json '%s': %s", path, exc) + return None + + if not isinstance(data, dict): + LOGGER.warning("json root in '%s' is not an object, skip", path) + return None + + return data + + +def _iter_json_records(input_dir: str) -> Iterable[dict[str, Any]]: + """Iterate over JSON files in the input directory and yield normalized records. + commit_sha/build_id/build_url are not set here; they are applied later only to + rows with the latest date (see _apply_build_metadata_to_latest_only). + """ + if not os.path.isdir(input_dir): + LOGGER.warning("input dir '%s' does not exist or is not a directory", input_dir) + return + + for entry in sorted(os.listdir(input_dir)): + if not entry.endswith(".json"): + continue + full_path = os.path.join(input_dir, entry) + if not os.path.isfile(full_path): + continue + + data = _load_json_file(full_path) + if data is None: + continue + + record: dict[str, Any] = dict(data) + record.setdefault("date", datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")) + record["source_file"] = os.path.basename(full_path) + yield record + + +def _collect_records(input_dir: str) -> list[dict[str, Any]]: + """Collect all JSON records into a list.""" + records: list[dict[str, Any]] = [] + for record in _iter_json_records(input_dir): + records.append(record) + return records + + +def _apply_build_metadata_to_latest_only( + records: Sequence[dict[str, Any]], + commit_sha: str | None, + build_id: str | None, + build_url: str | None, +) -> None: + """Set commit_sha, build_id, build_url only on rows with the latest date. + Other rows get None so that build info is not duplicated for older benchmark data. + """ + if not records: + return + max_date = max((r.get("date") or "") for r in records) + for r in records: + if (r.get("date") or "") == max_date: + r["commit_sha"] = commit_sha + r["build_id"] = build_id + r["build_url"] = build_url + else: + r["commit_sha"] = None + r["build_id"] = None + r["build_url"] = None + + +def _sort_records_for_summary(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Sort so that same model_id is grouped, newest date first within each group.""" + by_date_desc = sorted(records, key=lambda r: (r.get("date") or ""), reverse=True) + return sorted(by_date_desc, key=lambda r: (r.get("model_id") or "")) + + +def _values_differ(a: Any, b: Any) -> bool: + """Return True if two values are considered different; avoid direct float equality.""" + if a is None and b is None: + return False + if a is None or b is None: + return True + if isinstance(a, float) and isinstance(b, float): + if a != a and b != b: + return False + if a != a or b != b: + return True + return abs(a - b) > 1e-9 + return a != b + + +def _apply_benchmark_change_highlight( + ws, + summary_columns: Sequence[str], + records: Sequence[dict[str, Any]], +) -> None: + """Grey cells in the latest row of each model when a benchmark metric changed vs previous date.""" + if not records: + return + col_to_index = {c: i + 1 for i, c in enumerate(summary_columns)} + # Walk by model_id blocks (records already sorted by model_id, date desc). + i = 0 + while i < len(records): + model_id = records[i].get("model_id") + block_start = i + while i < len(records) and records[i].get("model_id") == model_id: + i += 1 + block_end = i + newest_idx = block_start + prev_idx = block_start + 1 if block_start + 1 < block_end else None + if prev_idx is None: + continue + excel_row = newest_idx + 2 + for col in BENCHMARK_COLUMNS: + if col not in col_to_index: + continue + cur_val = records[newest_idx].get(col) + prev_val = records[prev_idx].get(col) + if _values_differ(cur_val, prev_val): + ws.cell(row=excel_row, column=col_to_index[col]).fill = GREY_BLOCK_FILL + + +def _build_raw_columns( + records: Sequence[dict[str, Any]], + summary_columns: Sequence[str], +) -> list[str]: + """Infer the column set for the raw sheet based on all records.""" + keys: set[str] = set() + for record in records: + keys.update(record.keys()) + # Ensure summary columns appear first; remaining columns sorted alphabetically. + ordered_keys: list[str] = [] + for key in summary_columns: + if key in keys: + ordered_keys.append(key) + keys.discard(key) + ordered_keys.extend(sorted(keys)) + return ordered_keys + + +def _to_float_if_numeric(value: Any) -> Any: + """Coerce to float when possible so Excel treats as number; avoid #### from narrow columns.""" + if value is None: + return value + if isinstance(value, (int, float)): + return float(value) if isinstance(value, int) else value + if isinstance(value, str): + try: + return float(value) + except (ValueError, TypeError): + return value + return value + + +def _write_sheet( + ws, + columns: Sequence[str], + rows: Iterable[dict[str, Any]], + numeric_columns: Sequence[str] = (), +) -> None: + """Write column names and row data into the given worksheet.""" + numeric_set = set(numeric_columns) + ws.append(list(columns)) + for record in rows: + row_values = [] + for col in columns: + v = record.get(col) + if col in numeric_set: + v = _to_float_if_numeric(v) + row_values.append(v) + ws.append(row_values) + + +def _format_benchmark_columns( + ws, + columns: Sequence[str], + num_data_rows: int, +) -> None: + """Set number format and column width for numeric benchmark columns so values display (no ####).""" + numeric_set = set(NUMERIC_FORMAT_COLUMNS) + for c, col_name in enumerate(columns): + if col_name not in numeric_set: + continue + col_letter = get_column_letter(c + 1) + ws.column_dimensions[col_letter].width = 14 + for r in range(2, 2 + num_data_rows): + cell = ws.cell(row=r, column=c + 1) + if cell.value is not None and isinstance(cell.value, (int, float)): + cell.number_format = "0.0000" + elif isinstance(cell.value, str): + try: + cell.value = float(cell.value) + cell.number_format = "0.0000" + except (ValueError, TypeError): + pass + + +def _ensure_parent_dir(path: str) -> None: + """Ensure that the parent directory of the output file exists.""" + parent = os.path.dirname(os.path.abspath(path)) + if not parent: + return + os.makedirs(parent, exist_ok=True) + + +def generate_excel_report( + input_dir: str, + output_file: str, + commit_sha: str | None, + build_id: str | None, + build_url: str | None, +) -> None: + """Main logic: load JSON records and generate an Excel report.""" + script_dir = os.path.dirname(os.path.abspath(__file__)) + summary_columns = _load_summary_columns(script_dir) + + records = _collect_records(input_dir) + if not records: + LOGGER.warning("no valid json records found under '%s'", input_dir) + + sorted_records = _sort_records_for_summary(records) + _apply_build_metadata_to_latest_only(sorted_records, commit_sha, build_id, build_url) + + wb = Workbook() + ws_summary = wb.active + ws_summary.title = "summary" + + _write_sheet(ws_summary, summary_columns, sorted_records, numeric_columns=NUMERIC_FORMAT_COLUMNS) + _format_benchmark_columns(ws_summary, summary_columns, len(sorted_records)) + _apply_benchmark_change_highlight(ws_summary, summary_columns, sorted_records) + + if sorted_records: + raw_columns = _build_raw_columns(sorted_records, summary_columns) + ws_raw = wb.create_sheet(title="raw") + _write_sheet(ws_raw, raw_columns, sorted_records) + + _ensure_parent_dir(output_file) + wb.save(output_file) + LOGGER.info("excel report saved to '%s'", output_file) + + +def main() -> None: + """Command-line entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + ) + + args = parse_args() + + commit_sha = args.commit_sha or os.getenv("BUILDKITE_COMMIT") + build_id = args.build_id or os.getenv("BUILDKITE_BUILD_ID") + build_url = args.build_url or os.getenv("BUILDKITE_BUILD_URL") + + generate_excel_report( + input_dir=args.input_dir, + output_file=args.output_file, + commit_sha=commit_sha, + build_id=build_id, + build_url=build_url, + ) + + +if __name__ == "__main__": + main() diff --git a/tools/nightly/send_nightly_perf_email.py b/tools/nightly/send_nightly_perf_email.py new file mode 100644 index 000000000..af11cc73a --- /dev/null +++ b/tools/nightly/send_nightly_perf_email.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +Send the nightly performance Excel report by email to DAILY_EMAIL_LIST. + +Reads SMTP and recipient config from environment variables. Use --dry-run to +validate config and print subject/body without sending. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import smtplib +import sys +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +LOGGER = logging.getLogger(__name__) + +# Env keys for SMTP and recipients. +ENV_SMTP_HOST = "SMTP_HOST" +ENV_SMTP_PORT = "SMTP_PORT" +ENV_SMTP_USERNAME = "SMTP_USERNAME" +ENV_SMTP_PASSWORD = "SMTP_PASSWORD" +ENV_DAILY_EMAIL_LIST = "DAILY_EMAIL_LIST" +ENV_EMAIL_SENDER = "EMAIL_SENDER" +ENV_EMAIL_SUBJECT_PREFIX = "EMAIL_SUBJECT_PREFIX" +ENV_BUILD_URL = "BUILDKITE_BUILD_URL" +ENV_COMMIT = "BUILDKITE_COMMIT" + +DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") + +# Do not attach Excel if size >= this (bytes); body will suggest downloading from build URL. +MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 + +SMTP_RETRIES = 3 +SMTP_RETRY_DELAY_SEC = 5 + + +def _get_required_env() -> dict[str, str]: + """Read required env vars; raise SystemExit with clear message if any missing.""" + required = { + ENV_SMTP_HOST: os.environ.get(ENV_SMTP_HOST), + ENV_SMTP_PORT: os.environ.get(ENV_SMTP_PORT), + ENV_SMTP_USERNAME: os.environ.get(ENV_SMTP_USERNAME), + ENV_SMTP_PASSWORD: os.environ.get(ENV_SMTP_PASSWORD), + ENV_DAILY_EMAIL_LIST: os.environ.get(ENV_DAILY_EMAIL_LIST), + } + missing = [k for k, v in required.items() if not (v and str(v).strip())] + if missing: + raise SystemExit(f"Missing required env vars: {', '.join(missing)}. Set them (e.g. in Buildkite secrets).") + return {k: str(v).strip() for k, v in required.items()} + + +def _get_latest_file(folder_path: str) -> str: + """Get the latest modified file from the folder path.""" + files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(".xlsx")] + if not files: + raise SystemExit(f"No Excel files found in {folder_path}") + return max(files, key=os.path.getmtime) + + +def _recipients_list(comma_separated: str) -> list[str]: + """Parse DAILY_EMAIL_LIST into a list of addresses.""" + return [a.strip() for a in comma_separated.split(",") if a.strip()] + + +def _build_body( + date_str: str, + commit_sha: str | None, + build_url: str | None, + attachment_skipped: bool, +) -> str: + """Plain-text body with build metadata; note when attachment was skipped.""" + lines = [ + f"Nightly performance report date: {date_str}", + "", + f"Commit: {commit_sha or 'N/A'}", + f"Build: {build_url or 'N/A'}", + "", + ] + if attachment_skipped: + lines.append( + "Report file was too large to attach. Please download the Excel from the build artifacts (Build URL above)." + ) + lines.append("") + return "\n".join(lines) + + +def _build_subject(prefix: str | None, date_str: str) -> str: + """Subject line: optional prefix + date.""" + base = f"Nightly Perf {date_str}" + if prefix and prefix.strip(): + return f"{prefix.strip()} {base}" + return base + + +def _send_mail( + report_file: str, + date_str: str, + dry_run: bool, +) -> None: + """Load config, build message, and send (or dry-run).""" + cfg = _get_required_env() + recipients = _recipients_list(cfg[ENV_DAILY_EMAIL_LIST]) + if not recipients: + raise SystemExit("DAILY_EMAIL_LIST is empty after parsing.") + + commit_sha = os.environ.get(ENV_COMMIT) + build_url = os.environ.get(ENV_BUILD_URL) + sender = os.environ.get(ENV_EMAIL_SENDER) or cfg[ENV_SMTP_USERNAME] + prefix = os.environ.get(ENV_EMAIL_SUBJECT_PREFIX) + + size = os.path.getsize(report_file) if os.path.isfile(report_file) else 0 + attach_excel = size < MAX_ATTACHMENT_BYTES and size > 0 + attachment_skipped = os.path.isfile(report_file) and not attach_excel + + body = _build_body( + commit_sha=commit_sha, build_url=build_url, date_str=date_str, attachment_skipped=attachment_skipped + ) + subject = _build_subject(prefix=prefix, date_str=date_str) + + msg = MIMEMultipart() + msg["Subject"] = subject + msg["From"] = sender + msg["To"] = ", ".join(recipients) + msg.attach(MIMEText(body, "plain", "utf-8")) + + if attach_excel: + with open(report_file, "rb") as f: + part = MIMEApplication(f.read(), _subtype="vnd.openxmlformats-officedocument.spreadsheetml.sheet") + part.add_header("Content-Disposition", "attachment", filename=os.path.basename(report_file)) + msg.attach(part) + + if dry_run: + LOGGER.info("dry-run: not sending mail") + print("To:", recipients, file=sys.stderr) + print("Subject:", subject, file=sys.stderr) + print("Attachment:", "yes" if attach_excel else "no (size limit)", file=sys.stderr) + print("Body preview:", body[:300] + ("..." if len(body) > 300 else ""), file=sys.stderr) + return + + port = int(cfg[ENV_SMTP_PORT], 10) + last_err: Exception | None = None + for attempt in range(SMTP_RETRIES): + try: + with smtplib.SMTP(cfg[ENV_SMTP_HOST], port=port, timeout=30) as smtp: + smtp.starttls() + smtp.login(cfg[ENV_SMTP_USERNAME], cfg[ENV_SMTP_PASSWORD]) + smtp.sendmail(sender, recipients, msg.as_string()) + LOGGER.info("sent nightly perf email to %d recipient(s)", len(recipients)) + return + except Exception as e: + last_err = e + LOGGER.warning("SMTP attempt %d/%d failed: %s", attempt + 1, SMTP_RETRIES, e) + if attempt < SMTP_RETRIES - 1: + import time + + time.sleep(SMTP_RETRY_DELAY_SEC) + raise SystemExit(f"Failed to send email after {SMTP_RETRIES} attempts.") from last_err + + +def _date_from_filename(path: str) -> str: + """Try to derive a date string from report filename (e.g. nightly_perf_20260211-020704.xlsx -> 2026-02-11).""" + base = os.path.splitext(os.path.basename(path))[0] + if base.startswith("nightly_perf_") and len(base) > 13: + raw = base.replace("nightly_perf_", "")[:8] + if len(raw) == 8 and raw.isdigit(): + return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}" + return base or "unknown" + + +def _vllm_omni_root() -> str: + """Resolve vllm-omni repo root: directory that contains a 'tests' subdir (and usually 'tools').""" + path = os.path.dirname(os.path.abspath(__file__)) + while path and path != os.path.dirname(path): + if os.path.isdir(os.path.join(path, "tests")): + return path + path = os.path.dirname(path) + return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + + +def _default_output_dir() -> str: + """Default: vllm-omni root / DEFAULT_OUTPUT_DIR (where performance .xlsx files live).""" + root = _vllm_omni_root() + return os.path.join(root, DEFAULT_OUTPUT_DIR) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Send nightly performance Excel report by email (config from env).", + ) + parser.add_argument( + "--report-file", + type=str, + default=_default_output_dir(), + help="Folder/file path to the nightly_perf_*.xlsx file; default is DEFAULT_OUTPUT_DIR.", + ) + parser.add_argument( + "--date", + type=str, + default=None, + help="Date string for subject/body (default: inferred from report filename).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print recipient, subject, and body; do not send.", + ) + return parser.parse_args() + + +def main() -> None: + """Entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + ) + args = parse_args() + + report_file = _get_latest_file(args.report_file) if os.path.isdir(args.report_file) else args.report_file + if not os.path.isfile(report_file): + raise SystemExit(f"Report file not found: {report_file}") + + date_str = args.date or _date_from_filename(report_file) + _send_mail(report_file=report_file, date_str=date_str, dry_run=args.dry_run) + + +if __name__ == "__main__": + main()