diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 013640988..7c8c4a89e 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -119,7 +119,7 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: ":full_moon: Omni Model Perf Test with H100" + - label: ":full_moon: Omni Model Perf Test & Test Case Statistics" key: nightly-performance timeout_in_minutes: 180 depends_on: upload-nightly-pipeline @@ -129,6 +129,8 @@ steps: - export BENCHMARK_DIR=tests - pytest -s -v tests/perf/scripts/run_benchmark.py - buildkite-agent artifact upload "tests/*.json" + - python tools/nightly/buildkite_testcase_statistics.py -o tests/buildkite_testcase_statistics.html + - buildkite-agent artifact upload "tests/*.html" agents: queue: "mithril-h100-pool" plugins: @@ -158,17 +160,19 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":email: Nightly Perf Collection & Email" + - label: ":email: Nightly Collection & Email" key: nightly-perf-distribution - depends_on: nightly-performance + depends_on: + - nightly-performance if: build.env("NIGHTLY") == "1" commands: - pip install openpyxl - export DEFAULT_INPUT_DIR=tests - export DEFAULT_OUTPUT_DIR=tests - buildkite-agent artifact download "tests/*.json" . --step nightly-performance + - buildkite-agent artifact download "tests/*.html" . --step nightly-performance - python tools/nightly/generate_nightly_perf_excel.py - - python tools/nightly/send_nightly_perf_email.py + - python tools/nightly/send_nightly_email.py --report-file "tests/*.xlsx, tests/*.html" - buildkite-agent artifact upload "tests/*.xlsx" agents: queue: "cpu_queue_premerge" diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py new file mode 100644 index 000000000..312bd6b6b --- /dev/null +++ b/tools/nightly/buildkite_testcase_statistics.py @@ -0,0 +1,798 @@ +#!/usr/bin/env python3 +""" +Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml; +collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report. + +Usage (run from repo root): + python scripts/buildkite_testcase_statistics.py -o buildkite_testcase_statistics.html + +Requires: PyYAML (pip install pyyaml) +""" + +from __future__ import annotations + +import argparse +import ast +import html +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +# Repo root (parent of the directory containing this script) +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +BUILDKITE_DIR = REPO_ROOT / ".buildkite" +PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml"] + + +def load_yaml(path: Path) -> dict: + with open(path, encoding="utf-8") as f: + return yaml.safe_load(f) + + +PYTEST_CMD_RE = re.compile(r"(?:timeout\s+\S+\s+)?(?:python3? -m\s+)?pytest\s+[^\n&|;]*") + + +def normalize_commands(step: dict) -> list[str]: + raw = step.get("commands") or [] + if isinstance(raw, str): + raw = [raw] + text = "\n".join((c.strip() if isinstance(c, str) else "") for c in raw if c) + text = text.replace("$$", "$") + result = [] + for m in PYTEST_CMD_RE.finditer(text): + line = m.group(0).strip() + line_start = text.rfind("\n", 0, m.start()) + 1 + before_match = text[line_start : m.start()] + if before_match.lstrip().startswith("#"): + continue + if line: + result.append(line) + return result + + +def extract_pytest_targets_from_line(line: str) -> list[str]: + """ + Parse pytest test targets (file path or marker expression) from a single command line. + Returns e.g. ['tests/xxx.py']. + When there is no explicit test file but only -m/--run-level, returns + ['-m expr'] or ['-m expr --run-level level'] with the full marker args. + """ + if "pytest" not in line: + return [] + # Strip leading timeout and anything before pytest + idx = line.find("pytest") + if idx == -1: + return [] + rest = line[idx:].strip() + if not rest.startswith("pytest"): + return [] + # Simple split on whitespace (no handling of quoted spaces) + parts = re.split(r"\s+", rest, maxsplit=1) + args = parts[1].strip() if len(parts) > 1 else "" + if not args: + return [] + targets = [] + # Match tests/.../*.py + for m in re.finditer(r"tests/[^\s'\"#]+\.py", args): + path = m.group(0) + if path not in targets: + targets.append(path) + if not targets: + # May be -m/--run-level only; record full marker arguments instead of a generic marker. + extra = _parse_extra_args_from_line(line) + if extra: + # Join into a single string, e.g. "-m core and cpu --run-level nightly" + targets.append(" ".join(extra)) + return targets + + +def get_pytest_targets_from_step(step: dict) -> list[tuple[str, str]]: + """ + Extract all pytest-related targets from a step. + Returns [(target, raw_line), ...] where target is tests/xxx.py or '(marker/run-level)'. + """ + lines = normalize_commands(step) + out = [] + seen = set() + for line in lines: + for t in extract_pytest_targets_from_line(line): + if t not in seen: + seen.add(t) + out.append((t, line)) + return out + + +def _parse_extra_args_from_line(raw_line: str) -> list[str]: + """Extract -m and --run-level args from a raw pytest command line. Strips extra spaces and quotes. + Correctly handles quoted expressions with spaces, e.g. -m 'core_model and cpu'. + """ + extra: list[str] = [] + + # -m: match -m followed by '...', "..." or a single token (so -m expression is one argv) + m = re.search(r"-m\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) + if m: + value = (m.group(1) or m.group(2) or m.group(3) or "").strip() + if value: + extra.extend(["-m", value]) + + r = re.search(r"--run-level\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) + if r: + value = (r.group(1) or r.group(2) or r.group(3) or "").strip() + if value: + extra.extend(["--run-level", value]) + + return extra + + +def _parse_collect_only_stdout(stdout: str, *, raise_on_empty: bool = True, stderr: str = "") -> list[str]: + """ + Parse pytest --collect-only -q stdout into a list of node ids. + Normalizes \\r\\n and \\r to \\n before splitting. + If raise_on_empty and no node ids found, raises RuntimeError; otherwise returns []. + """ + raw = (stdout or "").replace("\r\n", "\n").replace("\r", "\n") + out: list[str] = [] + for line in raw.splitlines(): + line = line.strip() + if not line or "::" not in line or "test_" not in line: + continue + out.append(line) + if not out and raise_on_empty: + raise RuntimeError( + f"Failed to parse pytest --collect-only output: no node ids found.\nstdout:\n{stdout}\n\nstderr:\n{stderr}" + ) + return out + + +def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_line: str) -> tuple[str, str, int]: + """ + Resolve target + repo_root + extra to (path_arg, fallback_path, timeout). + Raises FileNotFoundError/RuntimeError on invalid config. + """ + if target.endswith(".py"): + path = (repo_root / target).resolve() + if not path.exists(): + raise FileNotFoundError(f"Pytest target path does not exist: {path}") + return str(path), target.replace("\\", "/"), 60 + if not extra: + raise RuntimeError(f"Failed to parse -m/--run-level from pytest line: {raw_line!r}") + if not (repo_root / "tests").exists(): + raise FileNotFoundError("tests/ directory not found under repo root") + return "tests/", "tests/", 120 + + +def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]: + """ + Run pytest --collect-only -q for a target and return test node id list (incl. parametrized). + target: path like tests/foo.py or marker string (e.g. "-m expr"). raw_line used for -m/--run-level. + """ + extra = _parse_extra_args_from_line(raw_line) + path_arg, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) + + cmd_quiet = [sys.executable, "-m", "pytest", path_arg, "--collect-only", "-q"] + cmd_quiet.extend(extra) + result = subprocess.run( + cmd_quiet, + cwd=repo_root, + capture_output=True, + text=True, + timeout=timeout_quiet, + ) + if result.returncode != 0: + raise RuntimeError(f"pytest --collect-only failed for {path_arg} with args {extra}: {result.stderr.strip()}") + print(f"pytest --collect-only success for {path_arg} with args {extra}: {result.stdout.strip()}") + return _parse_collect_only_stdout(result.stdout or "", stderr=result.stderr or "") + + +def _extra_with_skip_marker(extra: list[str]) -> list[str]: + """Build extra args that add -m skip: if extra has -m expr then use 'expr and skip', else -m skip.""" + out: list[str] = [] + i = 0 + has_m = False + while i < len(extra): + if extra[i] == "-m" and i + 1 < len(extra): + out.extend(["-m", extra[i + 1] + " and skip"]) + has_m = True + i += 2 + elif extra[i] == "--run-level" and i + 1 < len(extra): + out.extend(["--run-level", extra[i + 1]]) + i += 2 + else: + i += 1 + if not has_m: + out = ["-m", "skip"] + out + return out + + +def get_skip_status(target: str, raw_line: str, repo_root: Path) -> set[str]: + """ + Use pytest -m skip --collect-only -q to collect tests marked with skip; return their node ids. + Does not run tests; only collects. Supports both pytest xxx.py and pytest -m "marker". + """ + extra = _parse_extra_args_from_line(raw_line) + try: + path_arg, _, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) + except (FileNotFoundError, RuntimeError): + return set() + + extra_skip = _extra_with_skip_marker(extra) + cmd = [sys.executable, "-m", "pytest", path_arg, "--collect-only", "-q"] + cmd.extend(extra_skip) + try: + result = subprocess.run( + cmd, + cwd=repo_root, + capture_output=True, + text=True, + timeout=timeout_quiet, + ) + except (subprocess.TimeoutExpired, FileNotFoundError): + return set() + if result.returncode != 0: + return set() + ids = _parse_collect_only_stdout(result.stdout or "", raise_on_empty=False) + return set(ids) + + +def get_docstring_for_node_id(repo_root: Path, node_id: str) -> str: + """ + Parse the docstring of the test_ function/method from source by pytest node id. + Supports path::test_func, path::TestClass::test_method, path::test_func[param], etc. + """ + if not node_id or "::" not in node_id or node_id.startswith("["): + return "" + parts = node_id.split("::") + # Path accepts forward slashes; no platform-specific replacement needed + path = (repo_root / parts[0]).resolve() + if not path.exists(): + return "" + # Strip parametrize suffix, e.g. test_foo[param] -> test_foo + func_part = parts[-1] + if "[" in func_part: + func_part = func_part.split("[", 1)[0] + try: + source = path.read_text(encoding="utf-8", errors="replace") + tree = ast.parse(source) + except Exception: + return "" + # parts: [file, maybe_class, func] or [file, func] + if len(parts) == 2: + # path::test_func + target_func = func_part + target_class = None + else: + # path::TestClass::test_method + target_class = parts[1] + target_func = func_part + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == target_func and target_class is None: + doc = ast.get_docstring(node) + return (doc or "").strip() + if isinstance(node, ast.ClassDef) and node.name == target_class: + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == target_func: + doc = ast.get_docstring(item) + return (doc or "").strip() + break + return "" + + +def get_test_file_for_node_id(node_id: str) -> str: + """Extract test file path from a pytest node id.""" + if not node_id: + return "" + if "::" in node_id: + return node_id.split("::", 1)[0] + if node_id.endswith(".py"): + return node_id + return "(marker-only / no collection)" + + +def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Path | None = None) -> None: + """Write stats to HTML with styling and responsive tables.""" + root = repo_root if repo_root is not None else REPO_ROOT + pipeline_badges = { + "test-ready": "ready", + "test-merge": "merge", + "test-nightly": "nightly", + } + + def esc(s: str) -> str: + return html.escape(str(s)) + + def tr_class(i: int) -> str: + return "odd" if i % 2 == 0 else "even" + + pipeline_totals: dict[str, int] = {} + for item in all_stats: + _, pipeline_name, _, _, _, _, count, _, _, _ = item + pipeline_totals[pipeline_name] = pipeline_totals.get(pipeline_name, 0) + count + pipeline_summary_rows = [] + for idx, (pipeline_name, pipeline_count) in enumerate(sorted(pipeline_totals.items())): + badge = pipeline_badges.get(pipeline_name, "") + badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) + pipeline_summary_rows.append( + f'{badge_span}{pipeline_count}' + ) + + # Summary rows by module (item: ..., names, docstrings, skipped_ids) + summary_rows = [] + for i, item in enumerate(all_stats): + _, pipeline_name, label, test_files, _, marker_targets, count, _, _, _ = item + if test_files: + files_html = "
".join(esc(f) for f in test_files) + elif marker_targets: + # Display full marker / run-level arguments when no explicit test files. + files_html = "
".join(esc(m) for m in marker_targets) + else: + files_html = "No explicit tests or markers" + badge = pipeline_badges.get(pipeline_name, "") + badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) + summary_rows.append( + f'' + f'{badge_span}{esc(label)}{count}' + f'{files_html}' + ) + + # Detail rows grouped by Pipeline, then by Test Suite (two-level collapsible) + case_details_thead = """ + + Pipeline + Test Suite + Test File + No. + Test Name + Status + Description + """ + # Per-item: (pipeline_name, label, count, badge_span, suite_rows_html) + pipeline_suites: dict[str, list[tuple[str, int, str, str]]] = {} # pipeline -> [(label, count, summary, body)] + for item in all_stats: + _, pipeline_name, label, _, _, _, count, names, docstrings, skipped_ids = item + badge = pipeline_badges.get(pipeline_name, "") + badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) + suite_summary = f"{esc(label)} ({count} cases)" + file_sections: list[str] = [] + if not names: + suite_rows = [ + f'' + f"{badge_span}{esc(label)}" + f'(marker-only / no collection)0' + f'No collection or only -m/--run-level' + f'' + ] + file_sections.append( + '
' + "(marker-only / no collection)" + f'
{case_details_thead}' + f"{''.join(suite_rows)}
" + ) + else: + file_groups: dict[str, list[tuple[str, str, bool]]] = {} + for idx, name in enumerate(names): + desc = (docstrings[idx] if idx < len(docstrings) else "") or "" + if not desc and "::" in name and not name.startswith("["): + desc = get_docstring_for_node_id(root, name) + file_key = get_test_file_for_node_id(name) + file_groups.setdefault(file_key, []).append((name, desc, name in skipped_ids)) + + for file_name in sorted(file_groups.keys()): + suite_rows = [] + for i, (name, desc, is_skipped) in enumerate(file_groups[file_name], 1): + desc_html = esc(desc).replace("\n", "
") if desc else "" + status_html = 'Skipped' if is_skipped else "—" + suite_rows.append( + f'' + f'{badge_span}{esc(label)}{esc(file_name)}' + f'{i}' + f'{esc(name)}' + f'{status_html}{desc_html}' + ) + file_sections.append( + f'
' + f"{esc(file_name)} ({len(file_groups[file_name])} cases)" + f'
{case_details_thead}' + f"{''.join(suite_rows)}
" + ) + body = "".join(file_sections) + if pipeline_name not in pipeline_suites: + pipeline_suites[pipeline_name] = [] + pipeline_suites[pipeline_name].append((label, count, suite_summary, body)) + + # Build two-level HTML: Pipeline (outer) -> Test Suite (inner) -> table + detail_sections_html_parts = [] + for pipeline_name in sorted(pipeline_badges.keys()): + if pipeline_name not in pipeline_suites: + continue + suites = pipeline_suites[pipeline_name] + pipeline_total = sum(c for _, c, _, _ in suites) + badge = pipeline_badges.get(pipeline_name, "") + badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) + pipeline_summary = f"{badge_span} ({pipeline_total} cases)" + inner_html = "".join( + f'
{suite_summary}' + f'
{body}
' + for _, _, suite_summary, body in suites + ) + detail_sections_html_parts.append( + f'
{pipeline_summary}' + f'
{inner_html}
' + ) + # Include pipelines that might be in all_stats but not in pipeline_badges (e.g. custom names) + for pipeline_name in sorted(pipeline_suites.keys()): + if pipeline_name in pipeline_badges: + continue + suites = pipeline_suites[pipeline_name] + pipeline_total = sum(c for _, c, _, _ in suites) + pipeline_summary = f"{esc(pipeline_name)} ({pipeline_total} cases)" + inner_html = "".join( + f'
{suite_summary}' + f'
{body}
' + for _, _, suite_summary, body in suites + ) + detail_sections_html_parts.append( + f'
{pipeline_summary}' + f'
{inner_html}
' + ) + detail_sections_html = "".join(detail_sections_html_parts) + + html_content = f""" + + + + + Buildkite Test Case Statistics + + + +
+

Buildkite Pytest Case Statistics

+

Test Suites: {len(all_stats)} · Total cases (sum of steps): {total} ·

+ +
+

Pipeline Summary

+
+ + + + + + {"".join(pipeline_summary_rows)} + +
PipelineTotal Cases
+
+
+ +
+

Summary by Test Suite

+
+ + + + + + {"".join(summary_rows)} + +
PipelineTest SuiteCasesTest Files
+
+
+ +
+

Case Details

+

Expand by Pipeline, then by Test Suite, then by Test File.

+
+ + + Filter applies to the Test File fold sections below. +
+ {detail_sections_html} +
+
+ ↑ Top + + + +""" + out_path.write_text(html_content, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Count pytest cases from Buildkite pipelines") + parser.add_argument( + "--buildkite-dir", + type=Path, + default=BUILDKITE_DIR, + help="Path to .buildkite directory", + ) + default_out = REPO_ROOT / "buildkite_testcase_statistics.html" + parser.add_argument( + "-o", + "--output", + type=Path, + default=default_out, + help=f"Output HTML file path (default: {default_out})", + ) + args = parser.parse_args() + + all_stats = [] + + for filename in PIPELINE_FILES: + yml_path = args.buildkite_dir / filename + if not yml_path.exists(): + print(f"Skipped (not found): {yml_path}", file=sys.stderr) + continue + data = load_yaml(yml_path) + steps = data.get("steps") or [] + pipeline_name = filename.replace(".yml", "") + + for step in steps: + if not isinstance(step, dict): + continue + label = step.get("label") or "(no label)" + # Skip commented-out steps (usually no label or commands) + if not step.get("commands"): + continue + targets = get_pytest_targets_from_step(step) + if not targets: + continue + + module_key = f"{pipeline_name} / {label}" + # test_files: only real .py files; marker-only steps will have an empty list. + test_files = [t for t, _ in targets if t.endswith(".py")] + # marker_targets: synthetic marker targets (e.g. "-m expr", "-m expr --run-level level"). + marker_targets = [t for t, _ in targets if not t.endswith(".py")] + # marker_only: all targets are marker expressions (no explicit .py files). + marker_only = bool(targets) and all(not t.endswith(".py") for t, _ in targets) + + names = [] + skipped_ids: set[str] = set() + for target, raw in targets: + collected = collect_test_names(target, REPO_ROOT, raw) + skipped_ids.update(get_skip_status(target, raw, REPO_ROOT)) + names.extend(collected) + count = len(names) - len(skipped_ids) + docstrings = [ + get_docstring_for_node_id(REPO_ROOT, n) if "::" in n and not n.startswith("[") else "" for n in names + ] + all_stats.append( + ( + module_key, + pipeline_name, + label, + test_files, + marker_only, + marker_targets, + count, + names, + docstrings, + skipped_ids, + ) + ) + + out_path = args.output.resolve() + out_path.parent.mkdir(parents=True, exist_ok=True) + total = sum(item[6] for item in all_stats) + write_html(all_stats, out_path, total, REPO_ROOT) + print(f"Written: {out_path}") + print(f"Test Suites: {len(all_stats)}, total cases (sum of steps): {total}") + + +if __name__ == "__main__": + main() diff --git a/tools/nightly/send_nightly_email.py b/tools/nightly/send_nightly_email.py new file mode 100644 index 000000000..723a5c3d5 --- /dev/null +++ b/tools/nightly/send_nightly_email.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +""" +Send the nightly report by email to DAILY_EMAIL_LIST. + +Reads SMTP and recipient config from environment variables. Use --dry-run to +validate config and print subject/body without sending. +""" + +from __future__ import annotations + +import argparse +import logging +import mimetypes +import os +import smtplib +import sys +from datetime import date +from email.header import Header +from email.mime.application import MIMEApplication +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +LOGGER = logging.getLogger(__name__) + +# Env keys for SMTP and recipients. +ENV_SMTP_HOST = "SMTP_HOST" +ENV_SMTP_PORT = "SMTP_PORT" +ENV_SMTP_USERNAME = "SMTP_USERNAME" +ENV_SMTP_PASSWORD = "SMTP_PASSWORD" +ENV_DAILY_EMAIL_LIST = "DAILY_EMAIL_LIST" +ENV_EMAIL_SENDER = "EMAIL_SENDER" +ENV_EMAIL_SUBJECT_PREFIX = "EMAIL_SUBJECT_PREFIX" +ENV_BUILD_URL = "BUILDKITE_BUILD_URL" +ENV_COMMIT = "BUILDKITE_COMMIT" + +DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") + +# Do not attach file if size >= this (bytes); body will suggest downloading from build URL. +MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 + +SMTP_RETRIES = 3 +SMTP_RETRY_DELAY_SEC = 5 + + +def _strip_quoted(s: str) -> str: + """Remove one layer of surrounding quote characters from env values. + + Handles both ASCII quotes ('", ''') and common Unicode quotes (“ ” ‘ ’). + """ + if not s: + return s + s = s.strip() + if not s: + return s + quotes = {'"', "'", "“", "”", "‘", "’"} + if len(s) >= 2 and s[0] in quotes and s[-1] in quotes: + s = s[1:-1].strip() + return s + + +def _get_required_env() -> dict[str, str]: + """Read required env vars; raise SystemExit with clear message if any missing.""" + required = { + ENV_SMTP_HOST: os.environ.get(ENV_SMTP_HOST), + ENV_SMTP_PORT: os.environ.get(ENV_SMTP_PORT), + ENV_SMTP_USERNAME: os.environ.get(ENV_SMTP_USERNAME), + ENV_SMTP_PASSWORD: os.environ.get(ENV_SMTP_PASSWORD), + ENV_DAILY_EMAIL_LIST: os.environ.get(ENV_DAILY_EMAIL_LIST), + } + missing = [k for k, v in required.items() if not (v and str(v).strip())] + if missing: + raise SystemExit(f"Missing required env vars: {', '.join(missing)}. Set them (e.g. in Buildkite secrets).") + return {k: _strip_quoted(str(v)) for k, v in required.items()} + + +def _get_latest_file(path: str) -> str | list[str]: + """ + Resolve report file patterns into concrete files. + + - If path is a directory: return all files in that directory (list). + - If path contains wildcards (e.g. *.xlsx, *.html): expand using glob and + return all matching files (list). + - If path is a plain filename or file path: treat as (dir, filename), + search that directory for files with the same basename and return the + latest modified one (str). + + Note: higher-level callers will turn a single str into [str] when they + want to attach multiple files. + """ + import glob + + # Support comma-/semicolon-separated list: e.g. "*.xlsx, *.html; reports/foo.xlsx" + parts = [p.strip() for p in path.split(";") for p in p.split(",") if p and p.strip()] + if len(parts) > 1: + files: list[str] = [] + for part in parts: + files_part = _get_latest_file(part) + if isinstance(files_part, str): + files.append(files_part) + else: + files.extend(files_part) + if not files: + raise SystemExit(f"No files resolved from list: {path}") + return sorted(set(files)) + + # Single expression after splitting + path = parts[0] if parts else path + + # Directory: return all files inside + if os.path.isdir(path): + files = [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] + if not files: + raise SystemExit(f"No files found in {path}") + return sorted(files) + + # Wildcard pattern: expand via glob + if any(ch in path for ch in ("*", "?", "[")) and not os.path.exists(path): + matches = [p for p in glob.glob(path) if os.path.isfile(p)] + if not matches: + raise SystemExit(f"No files match pattern {path!r}") + return sorted(matches) + + # Plain filename or explicit file path: search its directory for same basename + parent = os.path.dirname(path) or "." + base = os.path.basename(path) + if not base: + raise SystemExit(f"Invalid path (no filename): {path}") + if not os.path.isdir(parent): + raise SystemExit(f"Directory not found: {parent}") + candidates = [ + p + for p in (os.path.join(parent, f) for f in os.listdir(parent)) + if os.path.basename(p) == base and os.path.isfile(p) + ] + if not candidates: + raise SystemExit(f"No file named {base!r} found in {parent}") + return max(candidates, key=os.path.getmtime) + + +def _recipients_list(comma_separated: str) -> list[str]: + """Parse DAILY_EMAIL_LIST into a list of addresses.""" + return [_strip_quoted(a.strip()) for a in comma_separated.split(",") if a and a.strip()] + + +def _build_body( + date_str: str, + commit_sha: str | None, + build_url: str | None, + attachment_skipped: bool, +) -> str: + """Plain-text body with build metadata; note when attachment was skipped.""" + lines = [ + f"Nightly report date: {date_str}", + "", + f"Commit: {commit_sha or 'N/A'}", + f"Build: {build_url or 'N/A'}", + "", + ] + if attachment_skipped: + lines.append( + "Some report file(s) were too large to attach. Please download from the build artifacts (Build URL above)." + ) + lines.append("") + return "\n".join(lines) + + +def _build_subject(prefix: str | None, date_str: str) -> str: + """Subject line: optional prefix + date.""" + base = f"Nightly Report {date_str}" + if prefix and prefix.strip(): + return f"{prefix.strip()} {base}" + return base + + +def _mime_type_and_subtype(file_path: str) -> tuple[str, str]: + """Return (maintype, subtype) for the file; fallback to application/octet-stream.""" + full_type, _ = mimetypes.guess_type(file_path, strict=False) + if not full_type: + return "application", "octet-stream" + parts = full_type.split("/", 1) + if len(parts) != 2: + return "application", "octet-stream" + return parts[0], parts[1] + + +def _attach_file(msg: MIMEMultipart, file_path: str, max_bytes: int) -> bool: + """ + Attach one file to msg with correct MIME type. Return True if attached, False if skipped (e.g. size). + """ + if not os.path.isfile(file_path): + return False + size = os.path.getsize(file_path) + if size <= 0 or size >= max_bytes: + return False + maintype, subtype = _mime_type_and_subtype(file_path) + name = os.path.basename(file_path) + try: + if maintype == "text": + with open(file_path, encoding="utf-8", errors="replace") as f: + part = MIMEText(f.read(), subtype, "utf-8") + else: + with open(file_path, "rb") as f: + part = MIMEApplication(f.read(), _subtype=subtype) + part.add_header("Content-Disposition", "attachment", filename=name) + msg.attach(part) + return True + except OSError: + return False + + +def _send_mail( + report_files: list[str], + date_str: str, + dry_run: bool, +) -> None: + """Load config, build message (attach each file with correct MIME type), and send (or dry-run).""" + cfg = _get_required_env() + recipients = _recipients_list(cfg[ENV_DAILY_EMAIL_LIST]) + if not recipients: + raise SystemExit("DAILY_EMAIL_LIST is empty after parsing.") + + commit_sha = os.environ.get(ENV_COMMIT) + build_url = os.environ.get(ENV_BUILD_URL) + # Allow EMAIL_SENDER / EMAIL_SUBJECT_PREFIX to be quoted; strip quotes if present. + raw_sender = os.environ.get(ENV_EMAIL_SENDER) + sender = _strip_quoted(raw_sender) if raw_sender else cfg[ENV_SMTP_USERNAME] + raw_prefix = os.environ.get(ENV_EMAIL_SUBJECT_PREFIX) + prefix = _strip_quoted(raw_prefix) if raw_prefix else None + + attachment_skipped = any(os.path.isfile(p) and os.path.getsize(p) >= MAX_ATTACHMENT_BYTES for p in report_files) + body = _build_body( + commit_sha=commit_sha, build_url=build_url, date_str=date_str, attachment_skipped=attachment_skipped + ) + subject = _build_subject(prefix=prefix, date_str=date_str) + + msg = MIMEMultipart() + msg["Subject"] = Header(subject, "utf-8") + msg["From"] = sender + msg["To"] = ", ".join(recipients) + msg.attach(MIMEText(body, "plain", "utf-8")) + + attached_count = 0 + for file_path in report_files: + if os.path.isfile(file_path) and os.path.getsize(file_path) >= MAX_ATTACHMENT_BYTES: + attachment_skipped = True + else: + if _attach_file(msg, file_path, MAX_ATTACHMENT_BYTES): + attached_count += 1 + elif os.path.isfile(file_path): + attachment_skipped = True + + if dry_run: + LOGGER.info("dry-run: not sending mail") + print("To:", recipients, file=sys.stderr) + print("Subject:", subject, file=sys.stderr) + print("Attachments:", attached_count, "(skipped large:", attachment_skipped, ")", file=sys.stderr) + print("Body preview:", body[:300] + ("..." if len(body) > 300 else ""), file=sys.stderr) + return + + host = cfg[ENV_SMTP_HOST] + port = int(cfg[ENV_SMTP_PORT], 10) + use_ssl = port == 465 + last_err: Exception | None = None + for attempt in range(SMTP_RETRIES): + try: + if use_ssl: + with smtplib.SMTP_SSL(host, port=port, timeout=30) as smtp: + smtp.login(cfg[ENV_SMTP_USERNAME], cfg[ENV_SMTP_PASSWORD]) + smtp.sendmail(sender, recipients, msg.as_string()) + else: + with smtplib.SMTP(host, port=port, timeout=30) as smtp: + smtp.starttls() + smtp.login(cfg[ENV_SMTP_USERNAME], cfg[ENV_SMTP_PASSWORD]) + smtp.sendmail(sender, recipients, msg.as_string()) + LOGGER.info("sent nightly email to %d recipient(s)", len(recipients)) + return + except Exception as e: + last_err = e + LOGGER.warning("SMTP attempt %d/%d failed: %s", attempt + 1, SMTP_RETRIES, e) + if attempt < SMTP_RETRIES - 1: + import time + + time.sleep(SMTP_RETRY_DELAY_SEC) + hint = "" + if last_err and "getaddrinfo" in str(last_err).lower(): + hint = f" (check SMTP_HOST={host!r} is correct and reachable, e.g. nslookup {host})" + raise SystemExit(f"Failed to send email after {SMTP_RETRIES} attempts.{hint}") from last_err + + +def _vllm_omni_root() -> str: + """Resolve vllm-omni repo root: directory that contains a 'tests' subdir (and usually 'tools').""" + path = os.path.dirname(os.path.abspath(__file__)) + while path and path != os.path.dirname(path): + if os.path.isdir(os.path.join(path, "tests")): + return path + path = os.path.dirname(path) + return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + + +def _default_output_dir() -> str: + """Default: vllm-omni root / DEFAULT_OUTPUT_DIR (where nightly report files live).""" + root = _vllm_omni_root() + subdir = DEFAULT_OUTPUT_DIR if (DEFAULT_OUTPUT_DIR and DEFAULT_OUTPUT_DIR.strip()) else "." + return os.path.join(root, subdir) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Send nightly report by email (config from env).", + ) + parser.add_argument( + "--report-file", + type=str, + default=_default_output_dir(), + help="Folder/file path to the sent email file; default is DEFAULT_OUTPUT_DIR.", + ) + parser.add_argument( + "--date", + type=str, + default=None, + help="Date string for subject/body (default: inferred from report filename).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print recipient, subject, and body; do not send.", + ) + return parser.parse_args() + + +def main() -> None: + """Entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + ) + args = parse_args() + + result = _get_latest_file(args.report_file) + report_files = result if isinstance(result, list) else [result] + report_files = [p for p in report_files if os.path.isfile(p)] + if not report_files: + raise SystemExit("No report file(s) found.") + + date_str = args.date or date.today().strftime("%Y-%m-%d") + _send_mail(report_files=report_files, date_str=date_str, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/tools/nightly/send_nightly_perf_email.py b/tools/nightly/send_nightly_perf_email.py deleted file mode 100644 index 9b4bf7ece..000000000 --- a/tools/nightly/send_nightly_perf_email.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -""" -Send the nightly performance Excel report by email to DAILY_EMAIL_LIST. - -Reads SMTP and recipient config from environment variables. Use --dry-run to -validate config and print subject/body without sending. -""" - -from __future__ import annotations - -import argparse -import logging -import os -import smtplib -import sys -from email.header import Header -from email.mime.application import MIMEApplication -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -LOGGER = logging.getLogger(__name__) - -# Env keys for SMTP and recipients. -ENV_SMTP_HOST = "SMTP_HOST" -ENV_SMTP_PORT = "SMTP_PORT" -ENV_SMTP_USERNAME = "SMTP_USERNAME" -ENV_SMTP_PASSWORD = "SMTP_PASSWORD" -ENV_DAILY_EMAIL_LIST = "DAILY_EMAIL_LIST" -ENV_EMAIL_SENDER = "EMAIL_SENDER" -ENV_EMAIL_SUBJECT_PREFIX = "EMAIL_SUBJECT_PREFIX" -ENV_BUILD_URL = "BUILDKITE_BUILD_URL" -ENV_COMMIT = "BUILDKITE_COMMIT" - -DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") - -# Do not attach Excel if size >= this (bytes); body will suggest downloading from build URL. -MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 - -SMTP_RETRIES = 3 -SMTP_RETRY_DELAY_SEC = 5 - - -def _get_required_env() -> dict[str, str]: - """Read required env vars; raise SystemExit with clear message if any missing.""" - required = { - ENV_SMTP_HOST: os.environ.get(ENV_SMTP_HOST), - ENV_SMTP_PORT: os.environ.get(ENV_SMTP_PORT), - ENV_SMTP_USERNAME: os.environ.get(ENV_SMTP_USERNAME), - ENV_SMTP_PASSWORD: os.environ.get(ENV_SMTP_PASSWORD), - ENV_DAILY_EMAIL_LIST: os.environ.get(ENV_DAILY_EMAIL_LIST), - } - missing = [k for k, v in required.items() if not (v and str(v).strip())] - if missing: - raise SystemExit(f"Missing required env vars: {', '.join(missing)}. Set them (e.g. in Buildkite secrets).") - return {k: str(v).strip() for k, v in required.items()} - - -def _get_latest_file(folder_path: str) -> str: - """Get the latest modified file from the folder path.""" - files = [os.path.join(folder_path, f) for f in os.listdir(folder_path) if f.endswith(".xlsx")] - if not files: - raise SystemExit(f"No Excel files found in {folder_path}") - return max(files, key=os.path.getmtime) - - -def _recipients_list(comma_separated: str) -> list[str]: - """Parse DAILY_EMAIL_LIST into a list of addresses.""" - return [a.strip() for a in comma_separated.split(",") if a.strip()] - - -def _build_body( - date_str: str, - commit_sha: str | None, - build_url: str | None, - attachment_skipped: bool, -) -> str: - """Plain-text body with build metadata; note when attachment was skipped.""" - lines = [ - f"Nightly performance report date: {date_str}", - "", - f"Commit: {commit_sha or 'N/A'}", - f"Build: {build_url or 'N/A'}", - "", - ] - if attachment_skipped: - lines.append( - "Report file was too large to attach. Please download the Excel from the build artifacts (Build URL above)." - ) - lines.append("") - return "\n".join(lines) - - -def _build_subject(prefix: str | None, date_str: str) -> str: - """Subject line: optional prefix + date.""" - base = f"Nightly Perf {date_str}" - if prefix and prefix.strip(): - return f"{prefix.strip()} {base}" - return base - - -def _send_mail( - report_file: str, - date_str: str, - dry_run: bool, -) -> None: - """Load config, build message, and send (or dry-run).""" - cfg = _get_required_env() - recipients = _recipients_list(cfg[ENV_DAILY_EMAIL_LIST]) - if not recipients: - raise SystemExit("DAILY_EMAIL_LIST is empty after parsing.") - - commit_sha = os.environ.get(ENV_COMMIT) - build_url = os.environ.get(ENV_BUILD_URL) - sender = os.environ.get(ENV_EMAIL_SENDER) or cfg[ENV_SMTP_USERNAME] - prefix = os.environ.get(ENV_EMAIL_SUBJECT_PREFIX) - - size = os.path.getsize(report_file) if os.path.isfile(report_file) else 0 - attach_excel = size < MAX_ATTACHMENT_BYTES and size > 0 - attachment_skipped = os.path.isfile(report_file) and not attach_excel - - body = _build_body( - commit_sha=commit_sha, build_url=build_url, date_str=date_str, attachment_skipped=attachment_skipped - ) - subject = _build_subject(prefix=prefix, date_str=date_str) - - msg = MIMEMultipart() - msg["Subject"] = Header(subject, "utf-8") - msg["From"] = sender - msg["To"] = ", ".join(recipients) - msg.attach(MIMEText(body, "plain", "utf-8")) - - if attach_excel: - with open(report_file, "rb") as f: - part = MIMEApplication(f.read(), _subtype="vnd.openxmlformats-officedocument.spreadsheetml.sheet") - part.add_header("Content-Disposition", "attachment", filename=os.path.basename(report_file)) - msg.attach(part) - - if dry_run: - LOGGER.info("dry-run: not sending mail") - print("To:", recipients, file=sys.stderr) - print("Subject:", subject, file=sys.stderr) - print("Attachment:", "yes" if attach_excel else "no (size limit)", file=sys.stderr) - print("Body preview:", body[:300] + ("..." if len(body) > 300 else ""), file=sys.stderr) - return - - port = int(cfg[ENV_SMTP_PORT], 10) - last_err: Exception | None = None - for attempt in range(SMTP_RETRIES): - try: - with smtplib.SMTP(cfg[ENV_SMTP_HOST], port=port, timeout=30) as smtp: - smtp.starttls() - smtp.login(cfg[ENV_SMTP_USERNAME], cfg[ENV_SMTP_PASSWORD]) - smtp.sendmail(sender, recipients, msg.as_string()) - LOGGER.info("sent nightly perf email to %d recipient(s)", len(recipients)) - return - except Exception as e: - last_err = e - LOGGER.warning("SMTP attempt %d/%d failed: %s", attempt + 1, SMTP_RETRIES, e) - if attempt < SMTP_RETRIES - 1: - import time - - time.sleep(SMTP_RETRY_DELAY_SEC) - raise SystemExit(f"Failed to send email after {SMTP_RETRIES} attempts.") from last_err - - -def _date_from_filename(path: str) -> str: - """Try to derive a date string from report filename (e.g. nightly_perf_20260211-020704.xlsx -> 2026-02-11).""" - base = os.path.splitext(os.path.basename(path))[0] - if base.startswith("nightly_perf_") and len(base) > 13: - raw = base.replace("nightly_perf_", "")[:8] - if len(raw) == 8 and raw.isdigit(): - return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}" - return base or "unknown" - - -def _vllm_omni_root() -> str: - """Resolve vllm-omni repo root: directory that contains a 'tests' subdir (and usually 'tools').""" - path = os.path.dirname(os.path.abspath(__file__)) - while path and path != os.path.dirname(path): - if os.path.isdir(os.path.join(path, "tests")): - return path - path = os.path.dirname(path) - return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) - - -def _default_output_dir() -> str: - """Default: vllm-omni root / DEFAULT_OUTPUT_DIR (where performance .xlsx files live).""" - root = _vllm_omni_root() - return os.path.join(root, DEFAULT_OUTPUT_DIR) - - -def parse_args() -> argparse.Namespace: - """Parse command-line arguments.""" - parser = argparse.ArgumentParser( - description="Send nightly performance Excel report by email (config from env).", - ) - parser.add_argument( - "--report-file", - type=str, - default=_default_output_dir(), - help="Folder/file path to the nightly_perf_*.xlsx file; default is DEFAULT_OUTPUT_DIR.", - ) - parser.add_argument( - "--date", - type=str, - default=None, - help="Date string for subject/body (default: inferred from report filename).", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Print recipient, subject, and body; do not send.", - ) - return parser.parse_args() - - -def main() -> None: - """Entrypoint.""" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", - ) - args = parse_args() - - report_file = _get_latest_file(args.report_file) if os.path.isdir(args.report_file) else args.report_file - if not os.path.isfile(report_file): - raise SystemExit(f"Report file not found: {report_file}") - - date_str = args.date or _date_from_filename(report_file) - _send_mail(report_file=report_file, date_str=date_str, dry_run=args.dry_run) - - -if __name__ == "__main__": - main()