103 lines
5.0 KiB
Python
103 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit an RB3 Spark attention-layer fallback against the FP32 source."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
from audit_spark_qualcomm_attention import (
|
|
FIXTURE_SHA256,
|
|
SOURCE_ONNX_SHA256,
|
|
compare,
|
|
nearest_rank,
|
|
read_outputs,
|
|
sha256,
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--device-name", default="Dragonwing RB3 Gen 2 Vision Kit")
|
|
parser.add_argument("--compute-unit", choices=("cpu", "npu"), required=True)
|
|
parser.add_argument("--compile-options", required=True)
|
|
for name in (
|
|
"source-onnx", "fixture", "source-cpu-output", "compile-report",
|
|
"inference-report", "profile-report", "device-output", "output-report",
|
|
):
|
|
parser.add_argument(f"--{name}", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
compile_report = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
|
inference = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
|
profile = json.loads(args.profile_report.read_text(encoding="utf-8"))
|
|
if any(report.get("status") != "SUCCESS" for report in (compile_report, inference, profile)):
|
|
raise RuntimeError("compile, inference and profile must all succeed")
|
|
if compile_report.get("source_model_id") != "mn14dw6zn" or compile_report.get("options") != args.compile_options:
|
|
raise RuntimeError("compile did not use the pinned FP32 Spark source and requested route")
|
|
target_id = compile_report["target_model_id"]
|
|
if inference.get("model_id") != target_id or profile.get("model_id") != target_id:
|
|
raise RuntimeError("inference/profile did not use the compiled artifact")
|
|
if inference.get("input_dataset_id") != "d7m83gjy2":
|
|
raise RuntimeError("inference did not use the pinned cross-device fixture")
|
|
for report in (compile_report, inference, profile):
|
|
if report.get("device", {}).get("name") != args.device_name:
|
|
raise RuntimeError("job did not use the exact requested device")
|
|
requested = f"--compute_unit {args.compute_unit}"
|
|
if inference.get("options") != requested or profile.get("options") != requested:
|
|
raise RuntimeError("inference/profile requested a different compute unit")
|
|
files = {
|
|
name: {"bytes": path.stat().st_size, "sha256": sha256(path)}
|
|
for name, path in {
|
|
"source_onnx": args.source_onnx,
|
|
"fixture": args.fixture,
|
|
"source_cpu_output": args.source_cpu_output,
|
|
"device_output": args.device_output,
|
|
}.items()
|
|
}
|
|
if files["source_onnx"]["sha256"] != SOURCE_ONNX_SHA256 or files["fixture"]["sha256"] != FIXTURE_SHA256:
|
|
raise RuntimeError("source or fixture differs from the pinned experiment")
|
|
output = read_outputs(args.device_output, hub=True)
|
|
source = read_outputs(args.source_cpu_output, hub=False)
|
|
execution = profile["profile"]
|
|
times_us = execution["execution_summary"]["all_inference_times"]
|
|
if not times_us or any(not isinstance(value, int) or value <= 0 for value in times_us):
|
|
raise RuntimeError("missing or invalid device samples")
|
|
units = Counter(row.get("compute_unit") for row in execution["execution_detail"])
|
|
if units[args.compute_unit.upper()] <= 0:
|
|
raise RuntimeError("profile has no rows attributed to the requested compute unit")
|
|
report = {
|
|
"scope": "one real Spark-X2.5 attention layer at one static 1024-token cache shape; no generation E2E",
|
|
"status": f"component_executed_on_{args.compute_unit}_quality_unqualified",
|
|
"device": profile["device"],
|
|
"compile_job_id": compile_report["job_id"],
|
|
"inference_job_id": inference["job_id"],
|
|
"profile_job_id": profile["job_id"],
|
|
"target_model_id": target_id,
|
|
"input_dataset_id": inference["input_dataset_id"],
|
|
"compile_options": args.compile_options,
|
|
"files": files,
|
|
"source_cpu_vs_device": compare(source, output),
|
|
"profile": {
|
|
"sample_count": len(times_us),
|
|
"sample_min_us": min(times_us),
|
|
"nearest_rank_p50_us": nearest_rank(times_us, 0.50),
|
|
"nearest_rank_p95_us": nearest_rank(times_us, 0.95),
|
|
"reported_estimated_inference_time_us": execution["execution_summary"]["estimated_inference_time"],
|
|
"reported_peak_memory_bytes": execution["execution_summary"]["estimated_inference_peak_memory"],
|
|
"compute_unit_row_counts": dict(units),
|
|
},
|
|
"limits": [
|
|
"No next-token or task-level numerical tolerance is established.",
|
|
"No complete prefill, KV/ring, sampling, continuous generation or Omni deployment is tested.",
|
|
"Hosted component latency is not a complete request or sustained thermal profile.",
|
|
],
|
|
}
|
|
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps({"status": report["status"], "profile": report["profile"]}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|