diff --git a/tools/nightly/buildkite_testcase_statistics.py b/tools/nightly/buildkite_testcase_statistics.py index 5f3217981..7a4f9bff3 100644 --- a/tools/nightly/buildkite_testcase_statistics.py +++ b/tools/nightly/buildkite_testcase_statistics.py @@ -1,10 +1,17 @@ #!/usr/bin/env python3 """ -Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml; -collect test cases (including parametrized) via pytest --collect-only -q and produce an HTML report. +Parse pytest commands from Buildkite test-ready.yml, test-merge.yml, test-nightly.yml, +and test-weekly.yml; collect test cases (including parametrized) via pytest --collect-only -q +and produce an HTML report. + +Leaf steps may live under ``group:`` blocks (nested ``steps``); those are flattened with the +group title carried into the report. Usage (run from repo root): - python scripts/buildkite_testcase_statistics.py -o buildkite_testcase_statistics.html + python tools/nightly/buildkite_testcase_statistics.py -o buildkite_testcase_statistics.html + python tools/nightly/buildkite_testcase_statistics.py --sample-preview -o buildkite_testcase_statistics_preview.html + +``--sample-preview`` emits placeholder rows only (no pytest / no real counts). Requires: PyYAML (pip install pyyaml) """ @@ -24,7 +31,7 @@ import yaml # Repo root (parent of the directory containing this script) REPO_ROOT = Path(__file__).resolve().parent.parent.parent BUILDKITE_DIR = REPO_ROOT / ".buildkite" -PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml"] +PIPELINE_FILES = ["test-ready.yml", "test-merge.yml", "test-nightly.yml", "test-weekly.yml"] def load_yaml(path: Path) -> dict: @@ -32,6 +39,30 @@ def load_yaml(path: Path) -> dict: return yaml.safe_load(f) +def iter_pytest_leaf_steps(steps: list, *, group: str | None = None) -> list[tuple[dict, str | None]]: + """ + Flatten Buildkite YAML steps, expanding ``group`` blocks that contain nested ``steps``. + + Returns ``(leaf_step_dict, group_title)`` for each leaf step that has non-empty ``commands``. + ``group_title`` is ``None`` for top-level steps outside any ``group:`` block. + """ + out: list[tuple[dict, str | None]] = [] + for raw in steps or []: + if not isinstance(raw, dict): + continue + nested = raw.get("steps") + if isinstance(nested, list) and nested: + next_group = group + if "group" in raw: + g = raw.get("group") + next_group = g if isinstance(g, str) else (str(g) if g is not None else group) + out.extend(iter_pytest_leaf_steps(nested, group=next_group)) + continue + if raw.get("commands"): + out.append((raw, group)) + return out + + PYTEST_CMD_RE = re.compile(r"(?:timeout\s+\S+\s+)?(?:python3? -m\s+)?pytest\s+[^\n&|;]*") @@ -55,10 +86,10 @@ def normalize_commands(step: dict) -> list[str]: def extract_pytest_targets_from_line(line: str) -> list[str]: """ - Parse pytest test targets (file path or marker expression) from a single command line. - Returns e.g. ['tests/xxx.py']. - When there is no explicit test file but only -m/--run-level, returns - ['-m expr'] or ['-m expr --run-level level'] with the full marker args. + Parse pytest test targets (file path, directory under tests/, or marker expression) from a single command line. + Returns e.g. ['tests/xxx.py'] or ['tests/e2e']. + When there is no explicit path but markers/options apply to the whole tree, returns + a synthetic token built from sidecar args (``--ignore``, ``-m``, ``-k``, ``--run-level``). """ if "pytest" not in line: return [] @@ -74,17 +105,33 @@ def extract_pytest_targets_from_line(line: str) -> list[str]: args = parts[1].strip() if len(parts) > 1 else "" if not args: return [] - targets = [] + targets: list[str] = [] + # Omit paths that only appear as --ignore / --test-config-file values when discovering roots. + sanitized_for_roots = _strip_test_config_file_clauses_from_args_fragment( + _strip_ignore_clauses_from_args_fragment(args) + ) # Match tests/.../*.py - for m in re.finditer(r"tests/[^\s'\"#]+\.py", args): + for m in re.finditer(r"tests/[^\s'\"#]+\.py", sanitized_for_roots): path = m.group(0) if path not in targets: targets.append(path) + # Directory / namespace roots: tests/e2e, tests/e2e/, tests/foo/bar (no .py suffix) + for m in re.finditer(r"\s(tests/[a-zA-Z0-9_./\-]+)(?=\s|$|[\"'])", " " + sanitized_for_roots): + tok = m.group(1).rstrip("/") + if tok.endswith(".py"): + continue + if tok not in targets: + targets.append(tok) + for m in re.finditer(r"['\"](tests/[a-zA-Z0-9_./\-]+)['\"]", sanitized_for_roots): + tok = m.group(1).rstrip("/") + if tok.endswith(".py"): + continue + if tok not in targets: + targets.append(tok) if not targets: - # May be -m/--run-level only; record full marker arguments instead of a generic marker. - extra = _parse_extra_args_from_line(line) + # Marker-only / option-driven selection relative to tests/ + extra = _pytest_collect_sidecar_args(line) if extra: - # Join into a single string, e.g. "-m core and cpu --run-level nightly" targets.append(" ".join(extra)) return targets @@ -93,40 +140,77 @@ def get_pytest_targets_from_step(step: dict) -> list[tuple[str, str]]: """ Extract all pytest-related targets from a step. Returns [(target, raw_line), ...] where target is tests/xxx.py or '(marker/run-level)'. + + Deduplication uses ``(target, raw_line)`` so the same script path with different options + (e.g. ``--test-config-file …``) stays distinct across Buildkite jobs. """ lines = normalize_commands(step) out = [] seen = set() for line in lines: for t in extract_pytest_targets_from_line(line): - if t not in seen: - seen.add(t) + key = (t, line) + if key not in seen: + seen.add(key) out.append((t, line)) return out -def _parse_extra_args_from_line(raw_line: str) -> list[str]: - """Extract -m and --run-level args from a raw pytest command line. Strips extra spaces and quotes. - Correctly handles quoted expressions with spaces, e.g. -m 'core_model and cpu'. +def _strip_ignore_clauses_from_args_fragment(fragment: str) -> str: + """Replace ``--ignore=…`` / ``--ignore …`` spans so path discovery does not treat ignores as roots.""" + return re.sub(r"--ignore(?:=\S+|\s+\S+)", " ", fragment) + + +def _strip_test_config_file_clauses_from_args_fragment(fragment: str) -> str: + """Replace ``--test-config-file`` spans so JSON paths under ``tests/`` are not mistaken for collection roots.""" + return re.sub(r"--test-config-file(?:=\S+|\s+\S+)", " ", fragment) + + +def _pytest_collect_sidecar_args(raw_line: str) -> list[str]: + """Extract pytest args that affect collection: ``--ignore``, ``--test-config-file``, ``-m``, + ``--run-level``, ``-k``. + + Order matches typical CI lines (ignores / perf driver config, then markers / run-level / keyword). + Correctly handles quoted ``-m`` / ``-k`` / ``--run-level`` values with spaces. """ extra: list[str] = [] - # -m: match -m followed by '...', "..." or a single token (so -m expression is one argv) - m = re.search(r"-m\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) - if m: - value = (m.group(1) or m.group(2) or m.group(3) or "").strip() + for m in re.finditer(r"--ignore(?:=(\S+)|\s+(\S+))", raw_line): + path = (m.group(1) or m.group(2) or "").strip() + if path: + extra.extend(["--ignore", path]) + + for m in re.finditer(r"--test-config-file(?:=(\S+)|\s+(\S+))", raw_line): + path = (m.group(1) or m.group(2) or "").strip() + if path: + extra.extend(["--test-config-file", path]) + + marker = re.search(r"-m\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) + if marker: + value = (marker.group(1) or marker.group(2) or marker.group(3) or "").strip() if value: extra.extend(["-m", value]) - r = re.search(r"--run-level\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) - if r: - value = (r.group(1) or r.group(2) or r.group(3) or "").strip() + runlvl = re.search(r"--run-level\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) + if runlvl: + value = (runlvl.group(1) or runlvl.group(2) or runlvl.group(3) or "").strip() if value: extra.extend(["--run-level", value]) + kw = re.search(r"-k\s+(?:'([^']*)'|\"([^\"]*)\"|(\S+))", raw_line) + if kw: + value = (kw.group(1) or kw.group(2) or kw.group(3) or "").strip() + if value: + extra.extend(["-k", value]) + return extra +def _parse_extra_args_from_line(raw_line: str) -> list[str]: + """Backward-compatible alias for :func:`_pytest_collect_sidecar_args`.""" + return _pytest_collect_sidecar_args(raw_line) + + def _parse_collect_only_stdout(stdout: str, *, raise_on_empty: bool = True, stderr: str = "") -> list[str]: """ Parse pytest --collect-only -q stdout into a list of node ids. @@ -167,8 +251,15 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l if not path.exists(): raise FileNotFoundError(f"Pytest target path does not exist: {path}") return [str(path)], target.replace("\\", "/"), 60 + + if " " not in target and not target.startswith("-") and target.startswith("tests/"): + rel_dir = Path(target.rstrip("/")) + dir_path = (repo_root / rel_dir).resolve() + if dir_path.exists() and dir_path.is_dir(): + return [str(dir_path)], target.replace("\\", "/").rstrip("/"), 120 + if not extra: - raise RuntimeError(f"Failed to parse -m/--run-level from pytest line: {raw_line!r}") + raise RuntimeError(f"Failed to parse pytest sidecar args from line: {raw_line!r}") if not (repo_root / "tests").exists(): raise FileNotFoundError("tests/ directory not found under repo root") return ["tests/"], "tests/", 120 @@ -177,9 +268,13 @@ def _resolve_pytest_target(target: str, repo_root: Path, extra: list[str], raw_l def collect_test_names(target: str, repo_root: Path, raw_line: str) -> list[str]: """ Run pytest --collect-only -q for a target and return test node id list (incl. parametrized). - target: path like tests/foo.py or marker string (e.g. "-m expr"). raw_line used for -m/--run-level. + + ``path_args`` come from ``target`` (file, dir under ``tests/``, or fallback ``tests/``). Sidecar + flags ``--ignore``, ``--test-config-file``, ``-m``, ``-k``, ``--run-level`` are taken from the + full ``raw_line`` and appended to the same invocation so collection matches CI (perf scripts + parametrize from ``--test-config-file`` at import time). """ - extra = _parse_extra_args_from_line(raw_line) + extra = _pytest_collect_sidecar_args(raw_line) path_args, _fallback, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) cmd_quiet = [sys.executable, "-m", "pytest", *path_args, "--collect-only", "-q"] @@ -210,6 +305,15 @@ def _extra_with_skip_marker(extra: list[str]) -> list[str]: elif extra[i] == "--run-level" and i + 1 < len(extra): out.extend(["--run-level", extra[i + 1]]) i += 2 + elif extra[i] == "-k" and i + 1 < len(extra): + out.extend(["-k", extra[i + 1]]) + i += 2 + elif extra[i] == "--ignore" and i + 1 < len(extra): + out.extend(["--ignore", extra[i + 1]]) + i += 2 + elif extra[i] == "--test-config-file" and i + 1 < len(extra): + out.extend(["--test-config-file", extra[i + 1]]) + i += 2 else: i += 1 if not has_m: @@ -222,7 +326,7 @@ def get_skip_status(target: str, raw_line: str, repo_root: Path) -> set[str]: Use pytest -m skip --collect-only -q to collect tests marked with skip; return their node ids. Does not run tests; only collects. Supports both pytest xxx.py and pytest -m "marker". """ - extra = _parse_extra_args_from_line(raw_line) + extra = _pytest_collect_sidecar_args(raw_line) try: path_args, _, timeout_quiet = _resolve_pytest_target(target, repo_root, extra, raw_line) except (FileNotFoundError, RuntimeError): @@ -308,6 +412,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa "test-ready": "ready", "test-merge": "merge", "test-nightly": "nightly", + "test-weekly": "weekly", } def esc(s: str) -> str: @@ -318,7 +423,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa pipeline_totals: dict[str, int] = {} for item in all_stats: - _, pipeline_name, _, _, _, _, count, _, _, _ = item + _, pipeline_name, _, _, _, _, _, count, _, _, _ = item pipeline_totals[pipeline_name] = pipeline_totals.get(pipeline_name, 0) + count pipeline_summary_rows = [] for idx, (pipeline_name, pipeline_count) in enumerate(sorted(pipeline_totals.items())): @@ -331,7 +436,8 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa # Summary rows by module (item: ..., names, docstrings, skipped_ids) summary_rows = [] for i, item in enumerate(all_stats): - _, pipeline_name, label, test_files, _, marker_targets, count, _, _, _ = item + _, pipeline_name, group, label, test_files, _, marker_targets, count, _, _, _ = item + group_cell = esc(group) if group else "—" if test_files: files_html = "
".join(esc(f) for f in test_files) elif marker_targets: @@ -343,14 +449,15 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) summary_rows.append( f'' - f'{badge_span}{esc(label)}{count}' + f'{badge_span}{group_cell}{esc(label)}{count}' f'{files_html}' ) - # Detail rows grouped by Pipeline, then by Test Suite (two-level collapsible) + # Detail rows grouped by Pipeline -> Buildkite group -> Test Suite -> files case_details_thead = """ Pipeline + Group Test Suite Test File No. @@ -358,18 +465,19 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa Status Description """ - # Per-item: (pipeline_name, label, count, badge_span, suite_rows_html) - pipeline_suites: dict[str, list[tuple[str, int, str, str]]] = {} # pipeline -> [(label, count, summary, body)] + # pipeline -> group_title -> [(label, count, suite_summary, body)] + pipeline_group_suites: dict[str, dict[str | None, list[tuple[str, int, str, str]]]] = {} for item in all_stats: - _, pipeline_name, label, _, _, _, count, names, docstrings, skipped_ids = item + _, pipeline_name, group, label, _, _, _, count, names, docstrings, skipped_ids = item + group_cell = esc(group) if group else "—" badge = pipeline_badges.get(pipeline_name, "") badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) suite_summary = f"{esc(label)} ({count} cases)" file_sections: list[str] = [] if not names: suite_rows = [ - f'' - f"{badge_span}{esc(label)}" + f'' + f"{badge_span}{group_cell}{esc(label)}" f'(marker-only / no collection)0' f'No collection or only -m/--run-level' f'' @@ -394,9 +502,11 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa for i, (name, desc, is_skipped) in enumerate(file_groups[file_name], 1): desc_html = esc(desc).replace("\n", "
") if desc else "" status_html = 'Skipped' if is_skipped else "—" + skip_attr = ' data-case-skipped="1"' if is_skipped else ' data-case-skipped="0"' suite_rows.append( - f'' - f'{badge_span}{esc(label)}{esc(file_name)}' + f'' + f"{badge_span}{group_cell}{esc(label)}" + f'{esc(file_name)}' f'{i}' f'{esc(name)}' f'{status_html}{desc_html}' @@ -408,44 +518,40 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa f"{''.join(suite_rows)}" ) body = "".join(file_sections) - if pipeline_name not in pipeline_suites: - pipeline_suites[pipeline_name] = [] - pipeline_suites[pipeline_name].append((label, count, suite_summary, body)) + pipeline_group_suites.setdefault(pipeline_name, {}).setdefault(group, []).append( + (label, count, suite_summary, body) + ) - # Build two-level HTML: Pipeline (outer) -> Test Suite (inner) -> table - detail_sections_html_parts = [] - for pipeline_name in sorted(pipeline_badges.keys()): - if pipeline_name not in pipeline_suites: + def _group_sort_key(k: str | None) -> tuple[bool, str]: + return (k is None, (k or "").lower()) + + detail_sections_html_parts: list[str] = [] + all_pipeline_keys = sorted(set(pipeline_badges.keys()) | set(pipeline_group_suites.keys())) + for pipeline_name in all_pipeline_keys: + if pipeline_name not in pipeline_group_suites: continue - suites = pipeline_suites[pipeline_name] - pipeline_total = sum(c for _, c, _, _ in suites) + groups_dict = pipeline_group_suites[pipeline_name] + pipeline_total = sum(c for suites in groups_dict.values() for (_, c, _, _) in suites) badge = pipeline_badges.get(pipeline_name, "") badge_span = f'{esc(pipeline_name)}' if badge else esc(pipeline_name) pipeline_summary = f"{badge_span} ({pipeline_total} cases)" - inner_html = "".join( - f'
{suite_summary}' - f'
{body}
' - for _, _, suite_summary, body in suites - ) + group_sections: list[str] = [] + for g_key in sorted(groups_dict.keys(), key=_group_sort_key): + suites = groups_dict[g_key] + group_total = sum(c for _, c, _, _ in suites) + display_g = "—" if g_key is None else g_key + inner_html = "".join( + f'
{suite_summary}' + f'
{body}
' + for _, _, suite_summary, body in suites + ) + group_sections.append( + f'
{esc(display_g)} ({group_total} cases)' + f'
{inner_html}
' + ) detail_sections_html_parts.append( f'
{pipeline_summary}' - f'
{inner_html}
' - ) - # Include pipelines that might be in all_stats but not in pipeline_badges (e.g. custom names) - for pipeline_name in sorted(pipeline_suites.keys()): - if pipeline_name in pipeline_badges: - continue - suites = pipeline_suites[pipeline_name] - pipeline_total = sum(c for _, c, _, _ in suites) - pipeline_summary = f"{esc(pipeline_name)} ({pipeline_total} cases)" - inner_html = "".join( - f'
{suite_summary}' - f'
{body}
' - for _, _, suite_summary, body in suites - ) - detail_sections_html_parts.append( - f'
{pipeline_summary}' - f'
{inner_html}
' + f'
{"".join(group_sections)}
' ) detail_sections_html = "".join(detail_sections_html_parts) @@ -466,6 +572,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa --ready: #3fb950; --merge: #a371f7; --nightly: #f0883e; + --weekly: #79c0ff; }} * {{ box-sizing: border-box; }} body {{ @@ -524,6 +631,20 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa border-color: var(--accent); box-shadow: 0 0 0 2px rgba(88, 166, 255, 0.18); }} + .filter-select {{ + min-width: 180px; + padding: 0.65rem 0.85rem; + border-radius: 8px; + border: 1px solid var(--border); + background: rgba(255,255,255,0.04); + color: var(--text); + outline: none; + font-size: 0.9rem; + }} + .filter-select:focus {{ + border-color: var(--accent); + box-shadow: 0 0 0 2px rgba(88, 166, 255, 0.18); + }} .filter-hint {{ color: var(--muted); font-size: 0.82rem; @@ -566,6 +687,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa .badge.ready {{ background: rgba(63, 185, 80, 0.2); color: var(--ready); }} .badge.merge {{ background: rgba(163, 113, 247, 0.2); color: var(--merge); }} .badge.nightly {{ background: rgba(240, 136, 62, 0.2); color: var(--nightly); }} + .badge.weekly {{ background: rgba(121, 192, 255, 0.2); color: var(--weekly); }} .badge.skip {{ background: rgba(139, 148, 158, 0.25); color: var(--muted); }} td.status {{ color: var(--muted); font-size: 0.85rem; }} .case-details details {{ @@ -595,6 +717,12 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa .case-details details th {{ border-top: 0; }} .case-details .pipeline-details {{ margin-bottom: 0.5rem; }} .case-details .pipeline-details .case-details-inner {{ padding-left: 0.5rem; margin-top: 0.25rem; }} + .case-details .group-details {{ margin-bottom: 0.35rem; margin-left: 0.35rem; }} + .case-details .group-details summary::before {{ + content: "▶ "; font-size: 0.7rem; color: var(--muted); margin-right: 0.35rem; + }} + .case-details .group-details[open] summary::before {{ content: "▼ "; }} + .case-details .group-details .group-details-inner {{ padding-left: 0.35rem; margin-top: 0.25rem; }} .case-details .suite-details {{ margin-bottom: 0.35rem; margin-left: 0.5rem; }} .case-details .suite-details summary::before {{ content: "▶ "; font-size: 0.7rem; color: var(--muted); margin-right: 0.35rem; @@ -663,7 +791,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa
- + {"".join(summary_rows)} @@ -674,7 +802,7 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa

Case Details

-

Expand by Pipeline, then by Test Suite, then by Test File.

+

Expand by Pipeline, then Buildkite group, then Test Suite, then by Test File.

- Filter applies to the Test File fold sections below. + + + Path filter applies to file sections; status hides non-matching rows.
{detail_sections_html}
@@ -692,28 +825,48 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa @@ -723,6 +876,86 @@ def write_html(all_stats: list[tuple], out_path: Path, total: int, repo_root: Pa out_path.write_text(html_content, encoding="utf-8") +def _sample_preview_stats() -> list[tuple]: + """Synthetic stats tuples matching ``main()`` layout — UI demo only.""" + names_a = [ + "tests/e2e/demo/test_alpha.py::test_smoke", + "tests/e2e/demo/test_alpha.py::test_params[x0]", + "tests/e2e/demo/test_beta.py::test_widgets", + ] + skipped_a = {"tests/e2e/demo/test_alpha.py::test_smoke"} + return [ + ( + "test-ready / :card_index_dividers: Simple Test / Simple · Diffusion", + "test-ready", + ":card_index_dividers: Simple Test", + "Simple · Diffusion & Model Executor Test", + ["tests/diffusion/test_dummy.py", "tests/model_executor/test_dummy.py"], + False, + [], + len(names_a) - len(skipped_a), + names_a, + ["Alpha smoke.", "", "Widgets."], + skipped_a, + ), + ( + "test-ready / Custom Pipeline Test", + "test-ready", + None, + "Custom Pipeline Test", + ["tests/e2e/offline_inference/custom_pipeline/test_cp.py"], + False, + [], + 1, + ["tests/e2e/offline_inference/custom_pipeline/test_cp.py::test_cp_one"], + [""], + set(), + ), + ( + "test-merge / :card_index_dividers: Diffusion Test / LoRA", + "test-merge", + ":card_index_dividers: Diffusion Test", + "Diffusion · LoRA Test", + ["tests/diffusion/lora/test_lora.py"], + False, + [], + 2, + [ + "tests/diffusion/lora/test_lora.py::test_load", + "tests/diffusion/lora/test_lora.py::test_infer[a]", + ], + ["", ""], + set(), + ), + ( + "test-nightly / Omni marker-only (sample)", + "test-nightly", + ":card_index_dividers: Omni Model Test", + "Omni · marker-only step (sample)", + [], + True, + ["-m full_model and omni and H100", "--run-level full_model"], + 0, + [], + [], + set(), + ), + ( + "test-weekly / Reliability (sample)", + "test-weekly", + None, + "Reliability Test - qwen3-omni", + ["tests/dfx/reliability/test_reliability_qwen3_omni.py"], + False, + [], + 1, + ["tests/dfx/reliability/test_reliability_qwen3_omni.py::test_placeholder"], + ["Weekly reliability placeholder."], + set(), + ), + ] + + def main() -> None: parser = argparse.ArgumentParser(description="Count pytest cases from Buildkite pipelines") parser.add_argument( @@ -739,8 +972,23 @@ def main() -> None: default=default_out, help=f"Output HTML file path (default: {default_out})", ) + parser.add_argument( + "--sample-preview", + action="store_true", + help="Write HTML using placeholder pipelines/groups/cases only (no pytest, no YAML-driven counts).", + ) args = parser.parse_args() + if args.sample_preview: + all_stats = _sample_preview_stats() + out_path = args.output.resolve() + out_path.parent.mkdir(parents=True, exist_ok=True) + total = sum(item[7] for item in all_stats) + write_html(all_stats, out_path, total, REPO_ROOT) + print(f"Written (sample preview): {out_path}") + print(f"Test Suites: {len(all_stats)}, total placeholder cases: {total}") + return + all_stats = [] for filename in PIPELINE_FILES: @@ -752,11 +1000,10 @@ def main() -> None: steps = data.get("steps") or [] pipeline_name = filename.replace(".yml", "") - for step in steps: + for step, group in iter_pytest_leaf_steps(steps): if not isinstance(step, dict): continue label = step.get("label") or "(no label)" - # Skip commented-out steps (usually no label or commands) if not step.get("commands"): continue targets = get_pytest_targets_from_step(step) @@ -764,6 +1011,8 @@ def main() -> None: continue module_key = f"{pipeline_name} / {label}" + if group: + module_key = f"{pipeline_name} / {group} / {label}" # test_files: only real .py files; marker-only steps will have an empty list. test_files = [t for t, _ in targets if t.endswith(".py")] # marker_targets: synthetic marker targets (e.g. "-m expr", "-m expr --run-level level"). @@ -785,6 +1034,7 @@ def main() -> None: ( module_key, pipeline_name, + group, label, test_files, marker_only, @@ -798,7 +1048,7 @@ def main() -> None: out_path = args.output.resolve() out_path.parent.mkdir(parents=True, exist_ok=True) - total = sum(item[6] for item in all_stats) + total = sum(item[7] for item in all_stats) write_html(all_stats, out_path, total, REPO_ROOT) print(f"Written: {out_path}") print(f"Test Suites: {len(all_stats)}, total cases (sum of steps): {total}")
PipelineTest SuiteCasesTest Files
PipelineGroupTest SuiteCasesTest Files