diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py
index 5f3217981..7a4f9bff3 100644
--- a/tools/nightly/buildkite_testcase_statistics.py
+++ b/tools/nightly/buildkite_testcase_statistics.py
@@ -1,10 +1,17 @@
#!/usr/bin/env python3
"""
-Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml;
-collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report.
+Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml,
+and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q
+and produce an HTML report.
+
+Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the
+group title carried into the report.
Usage (run from repo root):
- python scripts/buildkite_testcase_statistics.py -o buildkite_testcase_statistics.html
+ python tools/nightly/buildkite_testcase_statistics.py -o buildkite_testcase_statistics.html
+ python tools/nightly/buildkite_testcase_statistics.py --sample-preview -o buildkite_testcase_statistics_preview.html
+
+``--sample-preview`` emits placeholder rows only (no pytest / no real counts).
Requires: PyYAML (pip install pyyaml)
"""
@@ -24,7 +31,7 @@ import yaml
# Repo root (parent of the directory containing this script)
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
BUILDKITE_DIR = REPO_ROOT / ".buildkite"
-PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml"]
+PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"]
def load_yaml(path: Path) -> dict:
@@ -32,6 +39,30 @@ def load_yaml(path: Path) -> dict:
return yaml.safe_load(f)
+def iter_pytest_leaf_steps(steps: list, *, group: str | None = None) -> list[tuple[dict, str | None]]:
+ """
+ Flatten Buildkite YAML steps, expanding ``group`` blocks that contain nested ``steps``.
+
+ Returns ``(leaf_step_dict, group_title)`` for each leaf step that has non-empty ``commands``.
+ ``group_title`` is ``None`` for top-level steps outside any ``group:`` block.
+ """
+ out: list[tuple[dict, str | None]] = []
+ for raw in steps or []:
+ if not isinstance(raw, dict):
+ continue
+ nested = raw.get("steps")
+ if isinstance(nested, list) and nested:
+ next_group = group
+ if "group" in raw:
+ g = raw.get("group")
+ next_group = g if isinstance(g, str) else (str(g) if g is not None else group)
+ out.extend(iter_pytest_leaf_steps(nested, group=next_group))
+ continue
+ if raw.get("commands"):
+ out.append((raw, group))
+ return out
+
+
PYTEST_CMD_RE = re.compile(r"(?:timeout\s+\S+\s+)?(?:python3? -m\s+)?pytest\s+[^\n&|;]*")
@@ -55,10 +86,10 @@ def normalize_commands(step: dict) -> list[str]:
def extract_pytest_targets_from_line(line: str) -> list[str]:
"""
- Parse pytest test targets (file path or marker expression) from a single command line.
- Returns e.g. ['tests/xxx.py'].
- When there is no explicit test file but only -m/--run-level, returns
- ['-m expr'] or ['-m expr --run-level level'] with the full marker args.
+ Parse pytest test targets (file path, directory under tests/, or marker expression) from a single command line.
+ Returns e.g. ['tests/xxx.py'] or ['tests/e2e'].
+ When there is no explicit path but markers/options apply to the whole tree, returns
+ a synthetic token built from sidecar args (``--ignore``, ``-m``, ``-k``, ``--run-level``).
"""
if "pytest" not in line:
return []
@@ -74,17 +105,33 @@ def extract_pytest_targets_from_line(line: str) -> list[str]:
args = parts[1].strip() if len(parts) > 1 else ""
if not args:
return []
- targets = []
+ targets: list[str] = []
+ # Omit paths that only appear as --ignore / --test-config-file values when discovering roots.
+ sanitized_for_roots = _strip_test_config_file_clauses_from_args_fragment(
+ _strip_ignore_clauses_from_args_fragment(args)
+ )
# Match tests/.../*.py
- for m in re.finditer(r"tests/[^\s'\"#]+\.py", args):
+ for m in re.finditer(r"tests/[^\s'\"#]+\.py", sanitized_for_roots):
path = m.group(0)
if path not in targets:
targets.append(path)
+ # Directory / namespace roots: tests/e2e, tests/e2e/, tests/foo/bar (no .py suffix)
+ for m in re.finditer(r"\s(tests/[a-zA-Z0-9_./\-]+)(?=\s|$|[\"'])", " " + sanitized_for_roots):
+ tok = m.group(1).rstrip("/")
+ if tok.endswith(".py"):
+ continue
+ if tok not in targets:
+ targets.append(tok)
+ for m in re.finditer(r"['\"](tests/[a-zA-Z0-9_./\-]+)['\"]", sanitized_for_roots):
+ tok = m.group(1).rstrip("/")
+ if tok.endswith(".py"):
+ continue
+ if tok not in targets:
+ targets.append(tok)
if not targets:
- # May be -m/--run-level only; record full marker arguments instead of a generic marker.
- extra = _parse_extra_args_from_line(line)
+ # Marker-only / option-driven selection relative to tests/
+ extra = _pytest_collect_sidecar_args(line)
if extra:
- # Join into a single string, e.g. "-m core and cpu --run-level nightly"
targets.append(" ".join(extra))
return targets
@@ -93,40 +140,77 @@ def get_pytest_targets_from_step(step: dict) -> list[tuple[str, str]]:
"""
Extract all pytest-related targets from a step.
Returns [(target, raw_line), ...] where target is tests/xxx.py or '(marker/run-level)'.
+
+ Deduplication uses ``(target, raw_line)`` so the same script path with different options
+ (e.g. ``--test-config-file …``) stays distinct across Buildkite jobs.
"""
lines = normalize_commands(step)
out = []
seen = set()
for line in lines:
for t in extract_pytest_targets_from_line(line):
- if t not in seen:
- seen.add(t)
+ key = (t, line)
+ if key not in seen:
+ seen.add(key)
out.append((t, line))
return out
-def _parse_extra_args_from_line(raw_line: str) -> list[str]:
- """Extract -m and --run-level args from a raw pytest command line. Strips extra spaces and quotes.
- Correctly handles quoted expressions with spaces, e.g. -m 'core_model and cpu'.
+def _strip_ignore_clauses_from_args_fragment(fragment: str) -> str:
+ """Replace ``--ignore=…`` / ``--ignore …`` spans so path discovery does not treat ignores as roots."""
+ return re.sub(r"--ignore(?:=\S+|\s+\S+)", " ", fragment)
+
+
+def _strip_test_config_file_clauses_from_args_fragment(fragment: str) -> str:
+ """Replace ``--test-config-file`` spans so JSON paths under ``tests/`` are not mistaken for collection roots."""
+ return re.sub(r"--test-config-file(?:=\S+|\s+\S+)", " ", fragment)
+
+
+def _pytest_collect_sidecar_args(raw_line: str) -> list[str]:
+ """Extract pytest args that affect collection: ``--ignore``, ``--test-config-file``, ``-m``,
+ ``--run-level``, ``-k``.
+
+ Order matches typical CI lines (ignores / perf driver config, then markers / run-level / keyword).
+ Correctly handles quoted ``-m`` / ``-k`` / ``--run-level`` values with spaces.
"""
extra: list[str] = []
- # -m: match -m followed by '...', "..." or a single token (so -m expression is one argv)
- m = re.search(r"-m\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line)
- if m:
- value = (m.group(1) or m.group(2) or m.group(3) or "").strip()
+ for m in re.finditer(r"--ignore(?:=(\S+)|\s+(\S+))", raw_line):
+ path = (m.group(1) or m.group(2) or "").strip()
+ if path:
+ extra.extend(["--ignore", path])
+
+ for m in re.finditer(r"--test-config-file(?:=(\S+)|\s+(\S+))", raw_line):
+ path = (m.group(1) or m.group(2) or "").strip()
+ if path:
+ extra.extend(["--test-config-file", path])
+
+ marker = re.search(r"-m\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line)
+ if marker:
+ value = (marker.group(1) or marker.group(2) or marker.group(3) or "").strip()
if value:
extra.extend(["-m", value])
- r = re.search(r"--run-level\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line)
- if r:
- value = (r.group(1) or r.group(2) or r.group(3) or "").strip()
+ runlvl = re.search(r"--run-level\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line)
+ if runlvl:
+ value = (runlvl.group(1) or runlvl.group(2) or runlvl.group(3) or "").strip()
if value:
extra.extend(["--run-level", value])
+ kw = re.search(r"-k\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line)
+ if kw:
+ value = (kw.group(1) or kw.group(2) or kw.group(3) or "").strip()
+ if value:
+ extra.extend(["-k", value])
+
return extra
+def _parse_extra_args_from_line(raw_line: str) -> list[str]:
+ """Backward-compatible alias for :func:`_pytest_collect_sidecar_args`."""
+ return _pytest_collect_sidecar_args(raw_line)
+
+
def _parse_collect_only_stdout(stdout: str, *, raise_on_empty: bool = True, stderr: str = "") -> list[str]:
"""
Parse pytest --collect-only -q stdout into a list of node ids.
@@ -167,8 +251,15 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l
if not path.exists():
raise FileNotFoundError(f"Pytest target path does not exist: {path}")
return [str(path)], target.replace("\\", "/"), 60
+
+ if " " not in target and not target.startswith("-") and target.startswith("tests/"):
+ rel_dir = Path(target.rstrip("/"))
+ dir_path = (repo_root / rel_dir).resolve()
+ if dir_path.exists() and dir_path.is_dir():
+ return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120
+
if not extra:
- raise RuntimeError(f"Failed to parse -m/--run-level from pytest line: {raw_line!r}")
+ raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}")
if not (repo_root / "tests").exists():
raise FileNotFoundError("tests/ directory not found under repo root")
return ["tests/"], "tests/", 120
@@ -177,9 +268,13 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l
def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]:
"""
Run pytest --collect-only -q for a target and return test node id list (incl. parametrized).
- target: path like tests/foo.py or marker string (e.g. "-m expr"). raw_line used for -m/--run-level.
+
+ ``path_args`` come from ``target`` (file, dir under ``tests/``, or fallback ``tests/``). Sidecar
+ flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the
+ full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts
+ parametrize from ``--test-config-file`` at import time).
"""
- extra = _parse_extra_args_from_line(raw_line)
+ extra = _pytest_collect_sidecar_args(raw_line)
path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line)
cmd_quiet = [sys.executable, "-m", "pytest", *path_args, "--collect-only", "-q"]
@@ -210,6 +305,15 @@ def _extra_with_skip_marker(extra: list[str]) -> list[str]:
elif extra[i] == "--run-level" and i + 1 < len(extra):
out.extend(["--run-level", extra[i + 1]])
i += 2
+ elif extra[i] == "-k" and i + 1 < len(extra):
+ out.extend(["-k", extra[i + 1]])
+ i += 2
+ elif extra[i] == "--ignore" and i + 1 < len(extra):
+ out.extend(["--ignore", extra[i + 1]])
+ i += 2
+ elif extra[i] == "--test-config-file" and i + 1 < len(extra):
+ out.extend(["--test-config-file", extra[i + 1]])
+ i += 2
else:
i += 1
if not has_m:
@@ -222,7 +326,7 @@ def get_skip_status(target: str, raw_line: str, repo_root: Path) -> set[str]:
Use pytest -m skip --collect-only -q to collect tests marked with skip; return their node ids.
Does not run tests; only collects. Supports both pytest xxx.py and pytest -m "marker".
"""
- extra = _parse_extra_args_from_line(raw_line)
+ extra = _pytest_collect_sidecar_args(raw_line)
try:
path_args, _, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line)
except (FileNotFoundError, RuntimeError):
@@ -308,6 +412,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa
"test-ready": "ready",
"test-merge": "merge",
"test-nightly": "nightly",
+ "test-weekly": "weekly",
}
def esc(s: str) -> str:
@@ -318,7 +423,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa
pipeline_totals: dict[str, int] = {}
for item in all_stats:
- _, pipeline_name, _, _, _, _, count, _, _, _ = item
+ _, pipeline_name, _, _, _, _, _, count, _, _, _ = item
pipeline_totals[pipeline_name] = pipeline_totals.get(pipeline_name, 0) + count
pipeline_summary_rows = []
for idx, (pipeline_name, pipeline_count) in enumerate(sorted(pipeline_totals.items())):
@@ -331,7 +436,8 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa
# Summary rows by module (item: ..., names, docstrings, skipped_ids)
summary_rows = []
for i, item in enumerate(all_stats):
- _, pipeline_name, label, test_files, _, marker_targets, count, _, _, _ = item
+ _, pipeline_name, group, label, test_files, _, marker_targets, count, _, _, _ = item
+ group_cell = esc(group) if group else "—"
if test_files:
files_html = "
".join(esc(f) for f in test_files)
elif marker_targets:
@@ -343,14 +449,15 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa
badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name)
summary_rows.append(
f'
{esc(name)}| Pipeline | Test Suite | Cases | Test Files | |
|---|---|---|---|---|
| Pipeline | Group | Test Suite | Cases | Test Files |