From a3ecde98e0001481a0ed258850a7176789f60c0a Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:33:46 +0800 Subject: [PATCH] [CI][Bugfix] Fix the error in generating the performance data table and add a fallback mechanism that prevents the result file from being generated when test case execution fails. (#2839) Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/test-nightly.yml | 64 ++++++++++++-- .../scripts/diffusion_result_template.json | 86 +++++++++++++++++++ .../perf/scripts/result_omni_template.json | 55 ++++++++++++ tests/dfx/perf/scripts/run_benchmark.py | 17 +++- .../perf/scripts/run_diffusion_benchmark.py | 12 ++- tests/test_generate_nightly_perf_excel.py | 71 +++++++++++++++ tests/test_generate_nightly_perf_html.py | 54 ++++++++++++ tools/nightly/generate_nightly_perf_excel.py | 16 ++++ tools/nightly/generate_nightly_perf_html.py | 50 +++++++---- 9 files changed, 394 insertions(+), 31 deletions(-) create mode 100644 tests/dfx/perf/scripts/diffusion_result_template.json create mode 100644 tests/dfx/perf/scripts/result_omni_template.json create mode 100644 tests/test_generate_nightly_perf_excel.py create mode 100644 tests/test_generate_nightly_perf_html.py diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index ac43b597d..02d8cced4 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -110,8 +110,12 @@ steps: timeout_in_minutes: 180 commands: - export BENCHMARK_DIR=tests/dfx/perf/results - - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json - - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT agents: queue: "mithril-h100-pool" plugins: @@ -177,8 +181,12 @@ steps: commands: - export BENCHMARK_DIR=tests/dfx/perf/results - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json - - buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT agents: queue: "mithril-h100-pool" plugins: @@ -398,6 +406,46 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level advanced_model + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE + value: "120" + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test" key: nightly-diffusion-x2iat-performance timeout_in_minutes: 180 @@ -417,11 +465,9 @@ steps: EXIT3=$$? pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json EXIT4=$$? - if [ $$EXIT1 -eq 0 ] || [ $$EXIT2 -eq 0 ] || [ $$EXIT3 -eq 0 ] || [ $$EXIT4 -eq 0 ]; then - buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" - buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" - fi - exit $$((EXIT1 | EXIT2 | EXIT3)) + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4)) agents: queue: "mithril-h100-pool" plugins: diff --git a/tests/dfx/perf/scripts/diffusion_result_template.json b/tests/dfx/perf/scripts/diffusion_result_template.json new file mode 100644 index 000000000..86bdf1bc7 --- /dev/null +++ b/tests/dfx/perf/scripts/diffusion_result_template.json @@ -0,0 +1,86 @@ +[ + { + "test_name": null, + "backend": null, + "timestamp": null, + "server_params": { + "model": null, + "serve_args": { + "enable-diffusion-pipeline-profiler": false + } + }, + "benchmark_params": { + "name": null, + "dataset": null, + "task": null, + "width": 0, + "height": 0, + "num-inference-steps": 0, + "num-prompts": 0, + "max-concurrency": 0, + "num-input-images": 0, + "enable-negative-prompt": false, + "baseline": { + "throughput_qps": 0, + "latency_mean": 0, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0 + } + }, + "result": { + "duration": 0, + "completed_requests": 0, + "failed_requests": 0, + "throughput_qps": 0, + "latency_mean": 0, + "latency_median": 0, + "latency_p99": 0, + "latency_p95": 0, + "latency_p50": 0, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0, + "peak_memory_mb_median": 0, + "stage_durations_mean": {}, + "stage_durations_p50": {}, + "stage_durations_p99": {}, + "backend": null, + "model": null, + "dataset": null, + "task": null + }, + "log_file": null, + "Model": null, + "Framework": null, + "Hardware": null, + "Deployment": null, + "Task": null, + "Dataset": null, + "resolution": null, + "Parallelism": null, + "max_concurrency": 0, + "Cache": null, + "Quantization": null, + "offload": null, + "compile": null, + "Attn_backend": null, + "num_inference_steps": 0, + "completed": 0, + "failed": 0, + "throughput_qps": 0, + "latency_mean": 0, + "latency_median": 0, + "latency_p99": 0, + "latency_p95": 0, + "latency_p50": 0, + "peak_memory_mb_max": 0, + "peak_memory_mb_mean": 0, + "peak_memory_mb_median": 0, + "stage_durations_mean": {}, + "stage_durations_p50": {}, + "stage_durations_p99": {}, + "commit_sha": null, + "build_id": null, + "build_url": null, + "source_file": null + } +] diff --git a/tests/dfx/perf/scripts/result_omni_template.json b/tests/dfx/perf/scripts/result_omni_template.json new file mode 100644 index 000000000..1d6132140 --- /dev/null +++ b/tests/dfx/perf/scripts/result_omni_template.json @@ -0,0 +1,55 @@ +{ + "date": null, + "endpoint_type": null, + "backend": null, + "label": null, + "model_id": null, + "tokenizer_id": null, + "num_prompts": 0, + "request_rate": null, + "burstiness": 0, + "max_concurrency": 0, + "duration": 0, + "completed": 0, + "failed": 0, + "total_input_tokens": 0, + "total_output_tokens": 0, + "request_throughput": 0, + "request_goodput": null, + "output_throughput": 0, + "total_token_throughput": 0, + "total_audio_duration_s": 0, + "total_audio_frames": 0, + "audio_throughput": 0, + "max_output_tokens_per_s": 0, + "max_concurrent_requests": 0, + "rtfx": 0, + "mean_ttft_ms": 0, + "median_ttft_ms": 0, + "p99_ttft_ms": 0, + "mean_tpot_ms": 0, + "median_tpot_ms": 0, + "p99_tpot_ms": 0, + "mean_itl_ms": 0, + "median_itl_ms": 0, + "p99_itl_ms": 0, + "mean_e2el_ms": 0, + "median_e2el_ms": 0, + "p99_e2el_ms": 0, + "mean_audio_rtf": 0, + "median_audio_rtf": 0, + "p99_audio_rtf": 0, + "mean_audio_ttfp_ms": 0, + "median_audio_ttfp_ms": 0, + "p99_audio_ttfp_ms": 0, + "mean_audio_duration_s": 0, + "median_audio_duration_s": 0, + "p99_audio_duration_s": 0, + "baseline": { + "mean_ttft_ms": 0, + "mean_audio_ttfp_ms": 0, + "mean_audio_rtf": 0 + }, + "random_input_len": 0, + "random_output_len": 0 +} diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index d5ef1b49e..bea46f684 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -45,6 +45,8 @@ if CONFIG_FILE_PATH is None: CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json" + STAGE_CONFIGS_DIR = Path(__file__).parent.parent / "stage_configs" test_params = create_unique_server_params(BENCHMARK_CONFIGS, STAGE_CONFIGS_DIR) @@ -64,7 +66,7 @@ def omni_server(request): print(f"Starting OmniServer with test: {test_name}, model: {model}") - server_args = ["--stage-init-timeout", "300", "--init-timeout", "900"] + server_args = ["--stage-init-timeout", "600", "--init-timeout", "900"] if stage_config_path: server_args = ["--stage-configs-path", stage_config_path] + server_args with OmniServer(model, server_args) as server: @@ -141,8 +143,17 @@ def run_benchmark( result_dir = "./" result_path = os.path.join(result_dir, result_filename) - with open(result_path, encoding="utf-8") as f: - result = json.load(f) + if not os.path.exists(result_path): + with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: + template_result: dict[str, Any] = json.load(f) + Path(result_path).parent.mkdir(parents=True, exist_ok=True) + with open(result_path, "w", encoding="utf-8") as f: + json.dump(template_result, f, ensure_ascii=False, indent=2) + print(f"Benchmark result file not generated, fallback to template: {result_path}") + result = template_result + else: + with open(result_path, encoding="utf-8") as f: + result = json.load(f) if baseline_config: result["baseline"] = _baseline_thresholds_for_step( diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 8eeeec8df..23efa8bb0 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -52,6 +52,7 @@ BENCHMARK_SCRIPT = str( _SESSION_TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M%S") _RESULT_LOCK = threading.Lock() _BRANCHPOINT_COMMIT_SHA: str | None = None +DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json" def _get_config_file_from_argv() -> str | None: @@ -563,10 +564,17 @@ def run_benchmark( if process.returncode != 0: tmp_result_file.unlink(missing_ok=True) - raise RuntimeError(f"Benchmark script exited with code {process.returncode}") + print(f"ERROR:Benchmark script exited with code {process.returncode}") if not tmp_result_file.exists(): - raise FileNotFoundError(f"Benchmark result file not found: {tmp_result_file}") + with open(DIFFUSION_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: + template_payload = json.load(f) + # Template schema is fixed and owned by this repo: + # ``diffusion_result_template.json`` is a one-item list and metrics live at [0]["result"]. + template_metrics: dict[str, Any] = template_payload[0]["result"] + with open(tmp_result_file, "w", encoding="utf-8") as f: + json.dump(template_metrics, f, ensure_ascii=False, indent=2) + print(f"Benchmark result file not generated, fallback to template: {tmp_result_file}") try: with open(tmp_result_file, encoding="utf-8") as f: diff --git a/tests/test_generate_nightly_perf_excel.py b/tests/test_generate_nightly_perf_excel.py new file mode 100644 index 000000000..9b05d6de0 --- /dev/null +++ b/tests/test_generate_nightly_perf_excel.py @@ -0,0 +1,71 @@ +import importlib.util +import json +import sys +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _load_excel_module(): + repo_root = Path(__file__).resolve().parents[1] + module_path = repo_root / "tools" / "nightly" / "generate_nightly_perf_excel.py" + spec = importlib.util.spec_from_file_location("generate_nightly_perf_excel", module_path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _cell_value_by_header(ws, header_name: str, row_idx: int = 2): + headers = [c.value for c in ws[1]] + col_idx = headers.index(header_name) + 1 + return ws.cell(row=row_idx, column=col_idx).value + + +def test_generate_excel_report_with_perf_templates(tmp_path: Path): + module = _load_excel_module() + repo_root = Path(__file__).resolve().parents[1] + perf_scripts_dir = repo_root / "tests" / "dfx" / "perf" / "scripts" + + omni_template_path = perf_scripts_dir / "result_omni_template.json" + diffusion_template_path = perf_scripts_dir / "diffusion_result_template.json" + + omni_record = json.loads(omni_template_path.read_text(encoding="utf-8")) + diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8")) + + input_dir = tmp_path / "input" + diffusion_input_dir = tmp_path / "diffusion_input" + input_dir.mkdir() + diffusion_input_dir.mkdir() + + # Keep file names compatible with parser conventions in generate_nightly_perf_excel.py + omni_result_file = input_dir / "result_test_perf_random_1_4_in2500_out900_20260415-185642.json" + diffusion_result_file = diffusion_input_dir / "diffusion_result_qwen_image_edit_20260415-193200.json" + omni_result_file.write_text(json.dumps(omni_record, ensure_ascii=False, indent=2), encoding="utf-8") + diffusion_result_file.write_text(json.dumps(diffusion_records, ensure_ascii=False, indent=2), encoding="utf-8") + + output_file = tmp_path / "nightly_perf.xlsx" + module.generate_excel_report( + input_dir=str(input_dir), + diffusion_input_dir=str(diffusion_input_dir), + output_file=str(output_file), + commit_sha="test_commit_sha", + build_id="test_build_id", + build_url="https://example.com/build/123", + ) + + assert output_file.exists() + + wb = load_workbook(output_file) + assert set(wb.sheetnames) >= {"omni_summary", "diffusion_summary", "omni_raw", "diffusion_raw"} + + ws_omni_raw = wb["omni_raw"] + baseline_cell = _cell_value_by_header(ws_omni_raw, "baseline") + assert baseline_cell == json.dumps(omni_record["baseline"], ensure_ascii=False, sort_keys=True) + + ws_omni_summary = wb["omni_summary"] + assert _cell_value_by_header(ws_omni_summary, "commit_sha") == "test_commit_sha" + assert _cell_value_by_header(ws_omni_summary, "build_id") == "test_build_id" diff --git a/tests/test_generate_nightly_perf_html.py b/tests/test_generate_nightly_perf_html.py new file mode 100644 index 000000000..4e77eb3ad --- /dev/null +++ b/tests/test_generate_nightly_perf_html.py @@ -0,0 +1,54 @@ +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _load_html_module(): + repo_root = Path(__file__).resolve().parents[1] + module_path = repo_root / "tools" / "nightly" / "generate_nightly_perf_html.py" + spec = importlib.util.spec_from_file_location("generate_nightly_perf_html", module_path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_generate_html_report_with_perf_templates(tmp_path: Path): + module = _load_html_module() + repo_root = Path(__file__).resolve().parents[1] + perf_scripts_dir = repo_root / "tests" / "dfx" / "perf" / "scripts" + + omni_template_path = perf_scripts_dir / "result_omni_template.json" + diffusion_template_path = perf_scripts_dir / "diffusion_result_template.json" + + omni_record = json.loads(omni_template_path.read_text(encoding="utf-8")) + diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8")) + + input_dir = tmp_path / "input" + diffusion_input_dir = tmp_path / "diffusion_input" + input_dir.mkdir() + diffusion_input_dir.mkdir() + + omni_result_file = input_dir / "result_test_perf_random_1_4_in2500_out900_20260415-185642.json" + diffusion_result_file = diffusion_input_dir / "diffusion_result_qwen_image_edit_20260415-193200.json" + omni_result_file.write_text(json.dumps(omni_record, ensure_ascii=False, indent=2), encoding="utf-8") + diffusion_result_file.write_text(json.dumps(diffusion_records, ensure_ascii=False, indent=2), encoding="utf-8") + + output_file = tmp_path / "nightly_perf_v2.html" + module.generate_html_report( + input_dir=str(input_dir), + diffusion_input_dir=str(diffusion_input_dir), + output_file=str(output_file), + ) + + assert output_file.exists() + html = output_file.read_text(encoding="utf-8") + assert "Nightly Performance Report" in html + assert "Omni records 1" in html + assert f"Diffusion records {len(diffusion_records)}" in html + assert "const DIFF_DATA =" in html diff --git a/tools/nightly/generate_nightly_perf_excel.py b/tools/nightly/generate_nightly_perf_excel.py index 4bb778531..6ba1d1eef 100644 --- a/tools/nightly/generate_nightly_perf_excel.py +++ b/tools/nightly/generate_nightly_perf_excel.py @@ -665,6 +665,21 @@ def _to_float_if_numeric(value: Any) -> Any: return value +def _to_excel_compatible(value: Any) -> Any: + """Convert non-scalar objects to JSON string for openpyxl cell values.""" + if isinstance(value, (dict, list, tuple)): + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + except (TypeError, ValueError): + return str(value) + if isinstance(value, set): + try: + return json.dumps(sorted(value), ensure_ascii=False) + except (TypeError, ValueError): + return str(value) + return value + + def _write_sheet( ws, columns: Sequence[str], @@ -680,6 +695,7 @@ def _write_sheet( v = record.get(col) if col in numeric_set: v = _to_float_if_numeric(v) + v = _to_excel_compatible(v) row_values.append(v) ws.append(row_values) diff --git a/tools/nightly/generate_nightly_perf_html.py b/tools/nightly/generate_nightly_perf_html.py index dd5ece790..a50a46255 100644 --- a/tools/nightly/generate_nightly_perf_html.py +++ b/tools/nightly/generate_nightly_perf_html.py @@ -17,7 +17,7 @@ from typing import Any LOGGER = logging.getLogger(__name__) _RESULT_JSON_PREFIX = "result_test_" -_DIFFUSION_JSON_PREFIX = "diffusion_perf_" +_DIFFUSION_JSON_PREFIXES = ("diffusion_perf_", "diffusion_result_") DEFAULT_INPUT_DIR = os.getenv("DEFAULT_INPUT_DIR") or "tests" DEFAULT_OUTPUT_DIR = os.getenv("DEFAULT_OUTPUT_DIR") or "tests" DEFAULT_DIFFUSION_INPUT_DIR = os.getenv("DIFFUSION_BENCHMARK_DIR") @@ -51,7 +51,7 @@ def _default_diffusion_input_dir(input_dir: str) -> str: return DEFAULT_DIFFUSION_INPUT_DIR if DEFAULT_DIFFUSION_INPUT_DIR else input_dir -def _load_json_file(path: str) -> dict[str, Any] | None: +def _load_json_file(path: str) -> dict[str, Any] | list[Any] | None: try: with open(path, encoding="utf-8") as f: data = json.load(f) @@ -59,8 +59,8 @@ def _load_json_file(path: str) -> dict[str, Any] | None: LOGGER.warning("failed to load json '%s': %s", path, exc) return None - if not isinstance(data, dict): - LOGGER.warning("json root in '%s' is not an object, skip", path) + if not isinstance(data, (dict, list)): + LOGGER.warning("json root in '%s' is not an object or list, skip", path) return None return data @@ -170,9 +170,10 @@ def _iter_omni_json_records(input_dir: str) -> Iterable[dict[str, Any]]: def _parse_diffusion_from_filename(filename: str) -> dict[str, Any]: name, ext = os.path.splitext(filename) - if ext != ".json" or not name.startswith(_DIFFUSION_JSON_PREFIX): + if ext != ".json" or not any(name.startswith(prefix) for prefix in _DIFFUSION_JSON_PREFIXES): return {} - core = name[len(_DIFFUSION_JSON_PREFIX) :] + matched_prefix = next(prefix for prefix in _DIFFUSION_JSON_PREFIXES if name.startswith(prefix)) + core = name[len(matched_prefix) :] parts = core.split("_") if len(parts) < 2: return {} @@ -195,7 +196,7 @@ def _iter_diffusion_json_records(input_dir: str) -> Iterable[dict[str, Any]]: return for entry in sorted(os.listdir(input_dir)): - if not entry.endswith(".json") or not entry.startswith(_DIFFUSION_JSON_PREFIX): + if not entry.endswith(".json") or not any(entry.startswith(prefix) for prefix in _DIFFUSION_JSON_PREFIXES): continue full_path = os.path.join(input_dir, entry) if not os.path.isfile(full_path): @@ -204,17 +205,32 @@ def _iter_diffusion_json_records(input_dir: str) -> Iterable[dict[str, Any]]: if data is None: continue - record: dict[str, Any] = dict(data) filename_meta = _parse_diffusion_from_filename(os.path.basename(full_path)) - if "date" not in record or not record.get("date"): - record["date"] = filename_meta.get("date") or datetime.now( - timezone.utc, - ).strftime("%Y%m%d-%H%M%S") - if "test_name" not in record or not record.get("test_name"): - if "test_name" in filename_meta: - record["test_name"] = filename_meta["test_name"] - record["source_file"] = os.path.basename(full_path) - yield record + if isinstance(data, dict): + records = [data] + elif isinstance(data, list): + records = [r for r in data if isinstance(r, dict)] + else: + records = [] + + if not records: + LOGGER.warning("diffusion json '%s' has no valid records, skip", full_path) + continue + + for record in records: + flat: dict[str, Any] = dict(record) + result = flat.pop("result", None) + if isinstance(result, dict): + flat.update(result) + if "date" not in flat or not flat.get("date"): + flat["date"] = filename_meta.get("date") or datetime.now( + timezone.utc, + ).strftime("%Y%m%d-%H%M%S") + if "test_name" not in flat or not flat.get("test_name"): + if "test_name" in filename_meta: + flat["test_name"] = filename_meta["test_name"] + flat["source_file"] = os.path.basename(full_path) + yield flat def _collect_omni_records(input_dir: str) -> list[dict[str, Any]]: