166 lines
8.1 KiB
Python
166 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit a real Spark attention-layer AI Hub result without claiming model E2E."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
|
|
OUTPUT_NAMES = ("hidden", "k_new_0", "v_new_0")
|
|
SOURCE_ONNX_SHA256 = "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb"
|
|
QUANTIZED_ZIP_SHA256 = "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553"
|
|
FIXTURE_SHA256 = "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed"
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for block in iter(lambda: source.read(8 << 20), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def read_outputs(path: Path, *, hub: bool) -> dict[str, np.ndarray]:
|
|
with np.load(path, allow_pickle=False) as data:
|
|
actual_names = tuple(data.files)
|
|
expected_names = tuple(f"output_{i}__0" for i in range(3)) if hub else OUTPUT_NAMES
|
|
if actual_names != expected_names:
|
|
raise ValueError(f"{path}: output names/order differ: {actual_names}")
|
|
values = {name: np.asarray(data[key]) for name, key in zip(OUTPUT_NAMES, expected_names)}
|
|
shapes = ((1, 1, 2048), (1, 2, 1, 256), (1, 2, 1, 256))
|
|
for name, shape in zip(OUTPUT_NAMES, shapes):
|
|
value = values[name]
|
|
if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
|
|
raise ValueError(f"{path}: {name} violates FP32 finite layer-output contract")
|
|
return values
|
|
|
|
|
|
def compare(reference: dict[str, np.ndarray], actual: dict[str, np.ndarray]) -> dict:
|
|
result = {}
|
|
for name in OUTPUT_NAMES:
|
|
left = reference[name].astype(np.float64).ravel()
|
|
right = actual[name].astype(np.float64).ravel()
|
|
if not np.linalg.norm(left):
|
|
raise ValueError(f"{name}: zero reference cannot define relative L2")
|
|
result[name] = {
|
|
"relative_l2": float(np.linalg.norm(left - right) / np.linalg.norm(left)),
|
|
"max_abs": float(np.max(np.abs(left - right))),
|
|
"cosine": float(np.dot(left, right) / (np.linalg.norm(left) * np.linalg.norm(right))),
|
|
}
|
|
return result
|
|
|
|
|
|
def nearest_rank(values: list[int], fraction: float) -> int:
|
|
ordered = sorted(values)
|
|
return ordered[math.ceil(fraction * len(ordered)) - 1]
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--device-name", required=True)
|
|
for name in (
|
|
"source-onnx", "quantized-zip", "fixture", "source-cpu-output",
|
|
"quantized-cpu-output", "s25-output", "s25-inference-report", "device-output", "compile-report",
|
|
"inference-report", "profile-report", "output-report",
|
|
):
|
|
parser.add_argument(f"--{name}", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
compile_report = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
|
inference = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
|
s25_inference = json.loads(args.s25_inference_report.read_text(encoding="utf-8"))
|
|
profile = json.loads(args.profile_report.read_text(encoding="utf-8"))
|
|
if compile_report.get("status") != "SUCCESS" or inference.get("status") != "SUCCESS" or profile.get("status") != "SUCCESS":
|
|
raise RuntimeError("QNN DLC compile, device inference and profile must all succeed")
|
|
if (compile_report.get("source_quantized_model_id") != "mmd5e3v3q"
|
|
or compile_report.get("options") != "--target_runtime qnn_dlc --quantize_io"):
|
|
raise RuntimeError("compile did not use the pinned quantized Spark layer and QNN DLC layout")
|
|
target = compile_report["target_model_id"]
|
|
if inference["model_id"] != target or profile["model_id"] != target:
|
|
raise RuntimeError("device inference/profile used a different compiled artifact")
|
|
if inference["input_dataset_id"] != "d7m83gjy2":
|
|
raise RuntimeError("device inference did not use the pinned cross-device fixture")
|
|
if (s25_inference.get("status") != "SUCCESS" or s25_inference.get("model_id") != "mnowl6xpq"
|
|
or s25_inference.get("input_dataset_id") != inference["input_dataset_id"]
|
|
or s25_inference.get("device", {}).get("name") != "Samsung Galaxy S25"):
|
|
raise RuntimeError("historical S25 output lacks matching model/device/dataset provenance")
|
|
if inference["device"]["name"] != args.device_name or profile["device"]["name"] != args.device_name:
|
|
raise RuntimeError("jobs did not execute on the exact requested device")
|
|
if inference["options"] != "--compute_unit npu" or profile["options"] != "--compute_unit npu":
|
|
raise RuntimeError("jobs did not request NPU placement")
|
|
|
|
execution = profile["profile"]
|
|
times_us = execution["execution_summary"]["all_inference_times"]
|
|
if not times_us or any(not isinstance(value, int) or value <= 0 for value in times_us):
|
|
raise RuntimeError("missing or invalid device samples")
|
|
units = Counter(row.get("compute_unit") for row in execution["execution_detail"])
|
|
if units["NPU"] <= 0:
|
|
raise RuntimeError("profile has no attributed NPU execution rows")
|
|
|
|
source = read_outputs(args.source_cpu_output, hub=False)
|
|
quantized = read_outputs(args.quantized_cpu_output, hub=False)
|
|
s25 = read_outputs(args.s25_output, hub=True)
|
|
device_output = read_outputs(args.device_output, hub=True)
|
|
files = {
|
|
name: {"bytes": path.stat().st_size, "sha256": sha256(path)}
|
|
for name, path in {
|
|
"source_onnx": args.source_onnx,
|
|
"quantized_onnx_zip": args.quantized_zip,
|
|
"fixture": args.fixture,
|
|
"source_cpu_output": args.source_cpu_output,
|
|
"quantized_cpu_output": args.quantized_cpu_output,
|
|
"s25_output": args.s25_output,
|
|
"device_output": args.device_output,
|
|
}.items()
|
|
}
|
|
for name, expected in {
|
|
"source_onnx": SOURCE_ONNX_SHA256,
|
|
"quantized_onnx_zip": QUANTIZED_ZIP_SHA256,
|
|
"fixture": FIXTURE_SHA256,
|
|
}.items():
|
|
if files[name]["sha256"] != expected:
|
|
raise RuntimeError(f"{name} hash differs from pinned layer artifact or fixture")
|
|
report = {
|
|
"scope": "one real Spark-X2.5 full-attention decoder layer at one static 1024-token cache shape; no generation E2E",
|
|
"status": "component_executed_on_npu_quality_unqualified",
|
|
"device": profile["device"],
|
|
"compile_job_id": compile_report["job_id"],
|
|
"inference_job_id": inference["job_id"],
|
|
"profile_job_id": profile["job_id"],
|
|
"target_model_id": target,
|
|
"input_dataset_id": inference["input_dataset_id"],
|
|
"s25_prior_inference_job_id": s25_inference["job_id"],
|
|
"files": files,
|
|
"source_vs_quantized_cpu": compare(source, quantized),
|
|
"quantized_cpu_vs_device_npu": compare(quantized, device_output),
|
|
"source_cpu_vs_device_npu": compare(source, device_output),
|
|
"s25_npu_vs_device_npu": compare(s25, device_output),
|
|
"profile": {
|
|
"sample_count": len(times_us),
|
|
"sample_min_us": min(times_us),
|
|
"nearest_rank_p50_us": nearest_rank(times_us, 0.50),
|
|
"nearest_rank_p95_us": nearest_rank(times_us, 0.95),
|
|
"reported_estimated_inference_time_us": execution["execution_summary"]["estimated_inference_time"],
|
|
"reported_peak_memory_bytes": execution["execution_summary"]["estimated_inference_peak_memory"],
|
|
"compute_unit_row_counts": dict(units),
|
|
},
|
|
"limits": [
|
|
"No task-level numerical tolerance or next-token quality gate is established.",
|
|
"Hub component jobs do not prove device-local prefill, KV/ring update, sampling, continuous generation or Omni deployment.",
|
|
"Profile sample latency is one component, not complete-request or sustained thermal performance.",
|
|
],
|
|
}
|
|
args.output_report.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps({"status": report["status"], "profile": report["profile"]}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|