Audit X Elite CPU vocoder and isolate TFLite conversion parity
This commit is contained in:
1 parent
e4a4477929
commit
ea373e3cbf
11 files changed
+5025
-6
No files matched your search
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit one hosted Qwen3-TTS talker decode step against pinned ONNX CPU tensors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SOURCE_SHA256 = "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
FIXTURE_SHA256 = "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
REFERENCE_SHA256 = "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
COMPILE_OPTIONS = "--target_runtime qnn_context_binary --quantize_full_type float16"
|
||||
INPUT_NAMES = ["x", "cos", "sin", "mask"] + [
|
||||
name for layer in range(28) for name in (f"k_cache_{layer}", f"v_cache_{layer}")
|
||||
]
|
||||
OUTPUT_NAMES = ["logits"] + [
|
||||
name for layer in range(28) for name in (f"k_new_{layer}", f"v_new_{layer}")
|
||||
]
|
||||
|
||||
|
||||
def file_record(path: Path) -> dict:
|
||||
with path.open("rb") as source:
|
||||
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
||||
return {"bytes": path.stat().st_size, "sha256": digest}
|
||||
|
||||
|
||||
def require_array(data: np.lib.npyio.NpzFile, name: str, shape: tuple[int, ...]) -> np.ndarray:
|
||||
value = np.asarray(data[name])
|
||||
if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
|
||||
raise ValueError(f"{name}: expected finite FP32 {shape}, got {value.shape}/{value.dtype}")
|
||||
return value
|
||||
|
||||
|
||||
def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
|
||||
original = reference.astype(np.float64).ravel()
|
||||
observed = actual.astype(np.float64).ravel()
|
||||
error = original - observed
|
||||
norm = float(np.linalg.norm(original))
|
||||
delta = float(np.linalg.norm(error))
|
||||
return {
|
||||
"relative_l2": delta / norm if norm else (0.0 if delta == 0 else None),
|
||||
"max_abs_error": float(np.max(np.abs(error))),
|
||||
"reference_rms": float(np.sqrt(np.mean(original * original))),
|
||||
"output_rms": float(np.sqrt(np.mean(observed * observed))),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-onnx", type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, required=True)
|
||||
parser.add_argument("--cpu-reference", type=Path, required=True)
|
||||
parser.add_argument("--compile-report", type=Path, required=True)
|
||||
parser.add_argument("--inference-report", type=Path, required=True)
|
||||
parser.add_argument("--device-output", type=Path, required=True)
|
||||
parser.add_argument("--output-report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must both be terminal successes")
|
||||
if compiled.get("source_model_id") != "mq3xre4rq" or compiled.get("options") != COMPILE_OPTIONS:
|
||||
raise ValueError("compile source or options differ from the pinned talker artifact")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference did not use the audited target artifact")
|
||||
if inferred.get("input_dataset_id") != "d7dw6pgy2" or inferred.get("options") != "--compute_unit npu":
|
||||
raise ValueError("inference fixture or requested compute unit differs")
|
||||
if inferred.get("device") != compiled.get("device"):
|
||||
raise ValueError("compile/inference device metadata differs")
|
||||
|
||||
files = {
|
||||
name: file_record(path)
|
||||
for name, path in {
|
||||
"source_onnx": args.source_onnx,
|
||||
"fixture": args.fixture,
|
||||
"cpu_reference": args.cpu_reference,
|
||||
"device_output": args.device_output,
|
||||
}.items()
|
||||
}
|
||||
for name, expected in {
|
||||
"source_onnx": SOURCE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"cpu_reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if files[name]["sha256"] != expected:
|
||||
raise ValueError(f"{name} hash differs from pinned source")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as fixture:
|
||||
if fixture.files != INPUT_NAMES:
|
||||
raise ValueError("fixture input names/order differ from the source contract")
|
||||
require_array(fixture, "x", (1, 1, 1024))
|
||||
for name in ("cos", "sin"):
|
||||
require_array(fixture, name, (1, 1, 1, 128))
|
||||
require_array(fixture, "mask", (1, 1, 1, 257))
|
||||
for name in INPUT_NAMES[4:]:
|
||||
require_array(fixture, name, (1, 8, 256, 128))
|
||||
|
||||
details = {}
|
||||
with np.load(args.cpu_reference, allow_pickle=False) as reference, np.load(
|
||||
args.device_output, allow_pickle=False
|
||||
) as output:
|
||||
if reference.files != OUTPUT_NAMES:
|
||||
raise ValueError("CPU reference output names/order differ from the source contract")
|
||||
expected_device_keys = [f"output_{index}__0" for index in range(len(OUTPUT_NAMES))]
|
||||
if output.files != expected_device_keys:
|
||||
raise ValueError("device output names/order differ from the compiled contract")
|
||||
for index, name in enumerate(OUTPUT_NAMES):
|
||||
shape = (1, 3072) if index == 0 else (1, 8, 1, 128)
|
||||
source_value = require_array(reference, name, shape)
|
||||
device_value = require_array(output, expected_device_keys[index], shape)
|
||||
details[name] = compare(source_value, device_value)
|
||||
if name == "logits":
|
||||
details[name]["cpu_top1"] = int(np.argmax(source_value[0]))
|
||||
details[name]["device_top1"] = int(np.argmax(device_value[0]))
|
||||
details[name]["top1_matches"] = bool(
|
||||
details[name]["cpu_top1"] == details[name]["device_top1"]
|
||||
)
|
||||
|
||||
report = {
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": inferred["device"],
|
||||
"compile_job_id": compiled["job_id"],
|
||||
"inference_job_id": inferred["job_id"],
|
||||
"source_model_id": compiled["source_model_id"],
|
||||
"target_model_id": compiled["target_model_id"],
|
||||
"input_dataset_id": inferred["input_dataset_id"],
|
||||
"requested_run_options": inferred["options"],
|
||||
"files": files,
|
||||
"tensor_comparison": details,
|
||||
"cache_max_relative_l2": max(
|
||||
details[name]["relative_l2"] for name in OUTPUT_NAMES[1:]
|
||||
),
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested.",
|
||||
],
|
||||
}
|
||||
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"status": report["status"],
|
||||
"device": report["device"]["name"],
|
||||
"logits": details["logits"],
|
||||
"cache_max_relative_l2": report["cache_max_relative_l2"],
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a pinned Qwen3-TTS Code2Wav TFLite binary on local LiteRT CPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import platform
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from ai_edge_litert.interpreter import Interpreter
|
||||
|
||||
from audit_qwen_tts_vocoder_qualcomm import compare, read_output
|
||||
|
||||
|
||||
TFLITE_SHA256 = "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd"
|
||||
FIXTURE_SHA256 = "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
REFERENCE_SHA256 = "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def cpu_model() -> str:
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.exists():
|
||||
for line in cpuinfo.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("model name"):
|
||||
return line.partition(":")[2].strip()
|
||||
return platform.processor()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("model", "fixture", "reference", "output", "report"):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
parser.add_argument("--threads", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if args.threads < 1:
|
||||
raise ValueError("threads must be positive")
|
||||
for name, expected in {
|
||||
"model": TFLITE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if sha256(getattr(args, name)) != expected:
|
||||
raise ValueError(f"{name} differs from the pinned artifact")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as data:
|
||||
if data.files != ["quantized"]:
|
||||
raise ValueError("unexpected input dataset layout")
|
||||
input_value = np.asarray(data["quantized"])
|
||||
if input_value.shape != (1, 512, 97) or input_value.dtype != np.float32:
|
||||
raise ValueError("expected one FP32 [1,512,97] input")
|
||||
|
||||
interpreter = Interpreter(model_path=str(args.model), num_threads=args.threads)
|
||||
interpreter.allocate_tensors()
|
||||
inputs = interpreter.get_input_details()
|
||||
outputs = interpreter.get_output_details()
|
||||
if len(inputs) != 1 or len(outputs) != 1 or inputs[0]["name"] != "quantized":
|
||||
raise ValueError("unexpected TFLite I/O contract")
|
||||
if tuple(inputs[0]["shape"]) != input_value.shape or inputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite input shape/type differs")
|
||||
if tuple(outputs[0]["shape"]) != (1, 48000) or outputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite output shape/type differs")
|
||||
|
||||
interpreter.set_tensor(inputs[0]["index"], input_value)
|
||||
started = time.monotonic()
|
||||
interpreter.invoke()
|
||||
elapsed = time.monotonic() - started
|
||||
actual = np.asarray(interpreter.get_tensor(outputs[0]["index"]))
|
||||
if actual.shape != (1, 48000) or actual.dtype != np.float32 or not np.isfinite(actual).all():
|
||||
raise ValueError("TFLite output violates finite waveform contract")
|
||||
np.savez_compressed(args.output, wav=actual)
|
||||
|
||||
source = read_output(args.reference, "wav")
|
||||
report = {
|
||||
"scope": "local x86-64 LiteRT CPU execution of one retained Code2Wav TFLite binary; no hosted-device timing or complete TTS",
|
||||
"status": "component_cpu_numeric_parity_on_one_fixture",
|
||||
"runtime": {"ai_edge_litert": importlib.metadata.version("ai-edge-litert"), "numpy": np.__version__},
|
||||
"host": {"system": platform.system(), "release": platform.release(), "machine": platform.machine(), "processor": cpu_model()},
|
||||
"threads": args.threads,
|
||||
"model_sha256": sha256(args.model),
|
||||
"fixture_sha256": sha256(args.fixture),
|
||||
"reference_sha256": sha256(args.reference),
|
||||
"output_sha256": sha256(args.output),
|
||||
"source_onnx_cpu_vs_litert_cpu": compare(source, actual),
|
||||
"single_invoke_seconds_not_a_profile": elapsed,
|
||||
"limits": [
|
||||
"Local x86-64 CPU execution cannot attest SA8775P or RB3 device CPU behavior.",
|
||||
"LiteRT CPU output is not evidence that the Android GPU delegate is correct.",
|
||||
"One fixed vocoder fixture has no listening-quality tolerance or complete-stream gate.",
|
||||
],
|
||||
}
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": report["status"], "comparison": report["source_onnx_cpu_vs_litert_cpu"]}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user