Profile MiniCPM-o speech-head step on AMD NPU

benchmarks/edge_harness/experiments/probe_minicpmo_amd_npu_speech_head.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Validate real-weight MiniCPM-o speech-head placement and one-step parity on AMD NPU."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import time
+from collections import Counter
+from pathlib import Path
+
+import numpy as np
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for block in iter(lambda: source.read(8 << 20), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def compare(expected: np.ndarray, actual: np.ndarray) -> dict:
+    if expected.shape != actual.shape:
+        raise ValueError("output tensor shape changed")
+    delta = actual.astype(np.float64) - expected.astype(np.float64)
+    return {
+        "finite": bool(np.isfinite(actual).all()),
+        "relative_l2": float(np.linalg.norm(delta) / max(
+            np.linalg.norm(expected.astype(np.float64)), 1e-12)),
+        "max_absolute_error": float(np.max(np.abs(delta))),
+    }
+
+
+def write_report(path: Path, report: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("model", "fixture", "reference", "ep-dir", "report", "profile-prefix"):
+        parser.add_argument(f"--{name}", type=Path, required=True)
+    for name in ("model", "fixture", "reference"):
+        parser.add_argument(f"--expected-{name}-sha256", required=True)
+    parser.add_argument("--cpu-only", action="store_true")
+    args = parser.parse_args()
+    import onnxruntime as ort
+
+    artifact_hashes = {name: sha256(getattr(args, name))
+                       for name in ("model", "fixture", "reference")}
+    for name, actual in artifact_hashes.items():
+        if actual != getattr(args, f"expected_{name}_sha256"):
+            raise ValueError(f"{name} hash changed")
+    report = {
+        "scope": "real-weight one-step MiniCPM-o 4.5 speech head, fixed 256-token cache; not complete MiniCPM-o",
+        "os": platform.platform(),
+        "onnxruntime_version": ort.__version__,
+        "artifact_sha256": artifact_hashes,
+        "status": "started",
+    }
+    write_report(args.report, report)
+    with np.load(args.fixture, allow_pickle=False) as fixture:
+        inputs = {name: np.ascontiguousarray(fixture[name]) for name in fixture.files}
+    with np.load(args.reference, allow_pickle=False) as source:
+        expected = {name: np.asarray(source[name]) for name in source.files}
+    if len(inputs) != 44 or len(expected) != 41:
+        raise ValueError("speech-head fixture/reference schema changed")
+    started = time.perf_counter()
+    cpu = ort.InferenceSession(str(args.model), providers=["CPUExecutionProvider"])
+    report["cpu_session_create_s"] = time.perf_counter() - started
+    input_names = [item.name for item in cpu.get_inputs()]
+    output_names = [item.name for item in cpu.get_outputs()]
+    if set(input_names) != set(inputs) or set(output_names) != set(expected):
+        raise ValueError("ONNX speech-head input/output names differ from pinned fixture")
+    started = time.perf_counter()
+    cpu_outputs = dict(zip(output_names, cpu.run(None, inputs), strict=True))
+    report["cpu_one_inference_s"] = time.perf_counter() - started
+    report["cpu_vs_retained_reference"] = {
+        name: compare(expected[name], cpu_outputs[name]) for name in output_names}
+    report["cpu_top1"] = int(np.argmax(cpu_outputs["logits"]))
+    cpu_cache_max = max(
+        values["relative_l2"] for name, values in report["cpu_vs_retained_reference"].items()
+        if name != "logits")
+    if (not all(values["finite"] for values in report["cpu_vs_retained_reference"].values())
+            or report["cpu_vs_retained_reference"]["logits"]["relative_l2"] > 1e-4
+            or cpu_cache_max > 1e-4):
+        raise ValueError("local CPU speech-head output differs from retained reference")
+    report["status"] = "cpu_reference_pass"
+    write_report(args.report, report)
+    del cpu
+    if args.cpu_only:
+        return
+
+    ep_dir = args.ep_dir.resolve(strict=True)
+    os.environ["PATH"] = str(ep_dir) + os.pathsep + os.environ.get("PATH", "")
+    if hasattr(os, "add_dll_directory"):
+        os.add_dll_directory(str(ep_dir))
+    ort.register_execution_provider_library(
+        "vitisai", str(ep_dir / "onnxruntime_vitisai_ep.dll"))
+    devices = [device for device in ort.get_ep_devices()
+               if device.ep_name == "vitisai" and str(device.device.type).endswith("NPU")]
+    report["vitisai_npu_device_count"] = len(devices)
+    report["status"] = "vitisai_session_creation_started"
+    write_report(args.report, report)
+    if not devices:
+        raise RuntimeError("VitisAI EP did not expose an NPU device")
+    options = ort.SessionOptions()
+    options.add_provider_for_devices(devices, {})
+    options.enable_profiling = True
+    options.profile_file_prefix = str(args.profile_prefix)
+    started = time.perf_counter()
+    session = ort.InferenceSession(str(args.model), sess_options=options)
+    report["vitisai_session_create_s"] = time.perf_counter() - started
+    report["status"] = "vitisai_session_created"
+    write_report(args.report, report)
+    started = time.perf_counter()
+    actual = dict(zip(output_names, session.run(None, inputs), strict=True))
+    report["vitisai_one_inference_s"] = time.perf_counter() - started
+    report["vitisai_vs_cpu"] = {
+        name: compare(cpu_outputs[name], actual[name]) for name in output_names}
+    report["vitisai_top1"] = int(np.argmax(actual["logits"]))
+    report["vitisai_cache_max_relative_l2"] = max(
+        values["relative_l2"] for name, values in report["vitisai_vs_cpu"].items()
+        if name != "logits")
+    profile = Path(session.end_profiling())
+    report["profile_file"] = str(profile)
+    events = json.loads(profile.read_text(encoding="utf-8"))
+    report["node_providers"] = dict(Counter(
+        (event.get("args") or {}).get("provider") for event in events
+        if event.get("cat") == "Node"))
+    report["status"] = (
+        "component_npu_numeric_pass"
+        if report["node_providers"].get("vitisai", 0) > 0
+        and report["vitisai_top1"] == report["cpu_top1"]
+        and all(values["finite"] for values in report["vitisai_vs_cpu"].values())
+        and report["vitisai_vs_cpu"]["logits"]["relative_l2"] <= 0.01
+        and report["vitisai_cache_max_relative_l2"] <= 0.01
+        else "component_not_qualified"
+    )
+    write_report(args.report, report)
+    print(json.dumps({"status": report["status"],
+                      "npu_nodes": report["node_providers"].get("vitisai", 0),
+                      "top1_match": report["vitisai_top1"] == report["cpu_top1"]}))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/results/e2e_expansion_20260924/README.md

@@ -125,6 +125,8 @@ A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary ma
 
 The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. These artifacts are numerically unqualified on the fixture. RB3 rejected the floating-point-input artifact. Explicit X Elite ONNX CPU and SA8775P TFLite CPU fallback compiles passed. X Elite CPU-requested inference preserved CPU top-1 token 80 with logits relative L2 2.07e-5; its 100-sample decode-step profile measured p50/p95 191.414/309.428 ms, all 1,744 execution-detail rows on CPU. SA8775P TFLite CPU-requested talker inference also preserved token 80 with logits relative L2 3.33e-5; its 100-sample decode-step profile measured p50/p95 52.635/76.925 ms, all 1,800 execution-detail rows on CPU. A new RB3 TFLite CPU-requested step preserved token 80 with logits relative L2 2.29e-5 and maximum new-cache relative L2 3.04e-5; its 100-sample profile measured p50/p95 138.022/139.642 ms with all 1,800 execution-detail rows on CPU. A new S24 TFLite CPU-requested talker step preserved token 80 with logits relative L2 3.33e-5 and maximum new-cache relative L2 3.41e-5; its 100-sample profile measured p50/p95 36.976/43.264 ms with all 1,800 execution-detail rows on CPU. No complete talker loop or TTS stream passed.
 
+A separate [HX370 AMD NPU MiniCPM-o speech-head attempt](evidence/minicpmo_amd_npu_speech_head/README.md) ran the real fixed-cache 20-layer ONNX step as one VitisAI NPU node. The local ORT CPU output was bitwise identical to the retained reference; NPU preserved top speech token 1867 but differed 2.172% on logits and up to 4.353% on new-cache tensors, failing the provisional component gate. The 420.34 s session build and one cold 39.22 ms step do not profile or qualify a complete MiniCPM-o stream.
+
 The [MiniCPM-o speech-head study](evidence/minicpmo_tts_qualcomm/README.md) recovered the real 20-layer ONNX source and a matching retained fixture. S24 and S25 FP16 context-binary outputs preserved the ONNX CPU top-1 speech token with 0.346%/0.360% logits relative L2 on one step. Their 100-sample component profiles had p50/p95 8.537/9.006 ms and 7.197/7.615 ms, with all 1,029 execution-detail rows on NPU for each. FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P NPU-requested inference preserved CPU top-1 token 1867 with 0.3568% logits relative L2; its 100-sample profile measured p50/p95 11.901/13.038 ms, all 944 execution-detail rows on NPU. X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2; its 100-sample profile measured p50/p95 9.905/11.037 ms, all 944 execution-detail rows on NPU. RB3 FP16 QNN inference failed at graph load despite compile success; an explicit TFLite CPU fallback same-fixture inference preserved top-1 token 1867 with logits relative L2 1.26e-6 versus ONNX CPU; its 100-sample CPU-requested profile measured p50/p95 59.518/71.347 ms with all 1,008 execution-detail rows on CPU. The complete multimodal model remains unqualified on those devices.
 
 The [native Windows Qwen3-TTS sustained analysis](evidence/qwen_tts_sustained_drift/README.md) finds a sharp playback throughput collapse around minute 26–27 in the historical 30-minute RTX 5090 Laptop stream. The last 17 serial requests all had simulated underruns with median RTF 1.331; the separate WSL run did not show that behavior. The revised local profiler recorded a completed current-source 30-minute repeat with device-wide NVML telemetry. The final 119 requests had median RTF 0.185 and three brief simulated underruns, so the prior collapse did not recur. The historical evidence and uncontrolled repeat conditions cannot identify its cause; underrun-free native Windows playback remains unqualified.

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/.gitattributes

@@ -0,0 +1,2 @@
+*profile*.json binary -eol
+*.log binary -eol

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/README.md

@@ -0,0 +1,46 @@
+# MiniCPM-o speech-head step on HX370 AMD NPU (2026-09-24)
+
+**Disposition: real component executed on NPU; numerical gate failed;
+NOT E2E.** This is one fixed-cache, real-weight 20-layer speech-head
+decode step, not the thinker, multimodal encoders, vocoder, streaming
+speech, or a complete MiniCPM-o request.
+
+The retained 776,341,465-byte ONNX artifact (SHA-256
+`78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229`)
+was recovered for the earlier [Qualcomm component study](../minicpmo_tts_qualcomm/README.md).
+Its exact checkpoint revision is not attested by that historical export;
+the file hash pins this experiment. The model and 44-input fixture remain
+outside Git under `/home/zhout/project/edge_infer/models/`.
+The fixture has a synthetic hidden activation and 20 K/V cache pairs at
+length 256. The graph emits 6,562 logits and 40 new-cache tensors.
+
+On HX370 native Windows 11 build 26200, AMD NPU driver
+`32.0.203.329`, ONNX Runtime `1.30.0`, and AMD VitisAI EP
+`1.8.63.0`, the [CPU control](cpu_control.json) reproduced all
+41 retained ORT CPU outputs **bitwise**. The
+[NPU attempt](npu_probe.json) then placed the complete step in one
+VitisAI node event with no CPU node events. It preserved top speech token
+**1867**, but logits differed **2.172% relative L2** and the worst
+new-cache tensor differed **4.353% relative L2** from that CPU control.
+Both exceed the probe's provisional 1% component gate. The source ONNX
+is FP32; [AMD documents](https://ryzenai.docs.amd.com/projects/WinML/en/latest/model_support.html)
+automatic BF16 conversion when compiling float models for this NPU.
+The profile does not independently reveal internal precision, so this
+is a possible cause, not a proven explanation.
+
+Session creation took **420.34 s** and one cold inference took
+**39.22 ms**. These are not warmup-adjusted or repeated latencies,
+and power was not measured. The exact model, fixture and reference hashes,
+driver, raw log and [placement profile](npu_profile_2026-09-24_20-20-16_148.json)
+are pinned in [audit_report.json](audit_report.json). The mixed-encoding
+[native log](npu_probe.log) is retained. The
+[probe](../../../../experiments/probe_minicpmo_amd_npu_speech_head.py)
+checks every input/output name, CPU parity, actual node provider,
+top token, logits and cache numerics.
+
+Next, test a same-checkpoint export or calibrated A16W8 artifact with
+representative hidden/cache values and a task-derived tolerance. A
+passing one-step component would still need continuous speech-head state,
+talker and vocoder handoff, cancellation, admission, memory, quality and
+whole MiniCPM-o behavior through Omni before this device/model cell
+could be qualified.

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/audit_report.json

@@ -0,0 +1,31 @@
+{
+  "date": "2026-09-24",
+  "host": "HX370 native Windows 11 build 26200; AMD integrated NPU",
+  "driver_version": "32.0.203.329",
+  "backend": "ONNX Runtime 1.30.0 with WindowsWorkload.EP.AMD.VitisAI.Framework 1.8.63.0",
+  "checkpoint_revision": "unattested by the retained historical ONNX export; artifact hash pins this component",
+  "precision": "FP32 source ONNX; AMD documents automatic BF16 conversion for float NPU models; compiled internal dtype not independently traced",
+  "input": "one fixed synthetic decode activation and 20-layer KV cache at length 256; batch 1, no concurrency",
+  "artifact_sha256": {
+    "model": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
+    "fixture": "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4",
+    "reference": "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
+  },
+  "cpu_reference_status": "all 41 outputs bitwise identical to retained ORT CPU reference",
+  "npu_node_events": 1,
+  "cpu_node_events": 0,
+  "cpu_top1": 1867,
+  "npu_top1": 1867,
+  "logits_relative_l2_vs_cpu": 0.021717823760628377,
+  "max_new_cache_relative_l2_vs_cpu": 0.04353096853260796,
+  "vitisai_session_create_s": 420.3435175999766,
+  "one_cold_inference_s": 0.03921710001304746,
+  "power_condition": "uncontrolled; no power measurement",
+  "status": "NPU_COMPONENT_EXECUTED_NUMERIC_GATE_FAILED_NOT_E2E",
+  "npu_report": "npu_probe.json",
+  "cpu_control": "cpu_control.json",
+  "raw_log": "npu_probe.log",
+  "raw_log_sha256": "e0ed0132746af3120a85b430aa1e9388f87f6b60cda2e5bb209384cfedf55ba9",
+  "profile": "npu_profile_2026-09-24_20-20-16_148.json",
+  "profile_sha256": "86ab333ff6e113bea416e5da4e49ae390273f66a3111fd38975d4cc655b9183f"
+}

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/cpu_control.json

@@ -0,0 +1,221 @@
+{
+  "scope": "real-weight one-step MiniCPM-o 4.5 speech head, fixed 256-token cache; not complete MiniCPM-o",
+  "os": "Windows-11-10.0.26200-SP0",
+  "onnxruntime_version": "1.30.0",
+  "artifact_sha256": {
+    "model": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
+    "fixture": "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4",
+    "reference": "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
+  },
+  "status": "cpu_reference_pass",
+  "cpu_session_create_s": 4.8765737999929115,
+  "cpu_one_inference_s": 0.024474099976941943,
+  "cpu_vs_retained_reference": {
+    "logits": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_0": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_0": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_1": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_1": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_2": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_2": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_3": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_3": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_4": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_4": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_5": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_5": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_6": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_6": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_7": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_7": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_8": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_8": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_9": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_9": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_10": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_10": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_11": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_11": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_12": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_12": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_13": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_13": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_14": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_14": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_15": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_15": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_16": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_16": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_17": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_17": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_18": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_18": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_19": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_19": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    }
+  },
+  "cpu_top1": 1867
+}

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/npu_probe.json

@@ -0,0 +1,437 @@
+{
+  "scope": "real-weight one-step MiniCPM-o 4.5 speech head, fixed 256-token cache; not complete MiniCPM-o",
+  "os": "Windows-11-10.0.26200-SP0",
+  "onnxruntime_version": "1.30.0",
+  "artifact_sha256": {
+    "model": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
+    "fixture": "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4",
+    "reference": "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
+  },
+  "status": "component_not_qualified",
+  "cpu_session_create_s": 4.730387800023891,
+  "cpu_one_inference_s": 0.02789869997650385,
+  "cpu_vs_retained_reference": {
+    "logits": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_0": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_0": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_1": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_1": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_2": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_2": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_3": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_3": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_4": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_4": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_5": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_5": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_6": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_6": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_7": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_7": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_8": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_8": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_9": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_9": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_10": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_10": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_11": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_11": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_12": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_12": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_13": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_13": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_14": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_14": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_15": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_15": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_16": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_16": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_17": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_17": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_18": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_18": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "k_new_19": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    },
+    "v_new_19": {
+      "finite": true,
+      "relative_l2": 0.0,
+      "max_absolute_error": 0.0
+    }
+  },
+  "cpu_top1": 1867,
+  "vitisai_npu_device_count": 1,
+  "vitisai_session_create_s": 420.3435175999766,
+  "vitisai_one_inference_s": 0.03921710001304746,
+  "vitisai_vs_cpu": {
+    "logits": {
+      "finite": true,
+      "relative_l2": 0.021717823760628377,
+      "max_absolute_error": 0.12033289670944214
+    },
+    "k_new_0": {
+      "finite": true,
+      "relative_l2": 0.010074650417885205,
+      "max_absolute_error": 0.06488895416259766
+    },
+    "v_new_0": {
+      "finite": true,
+      "relative_l2": 0.009257562587801211,
+      "max_absolute_error": 0.017855048179626465
+    },
+    "k_new_1": {
+      "finite": true,
+      "relative_l2": 0.01337781098405101,
+      "max_absolute_error": 0.06293153762817383
+    },
+    "v_new_1": {
+      "finite": true,
+      "relative_l2": 0.01971893480027639,
+      "max_absolute_error": 0.006307706236839294
+    },
+    "k_new_2": {
+      "finite": true,
+      "relative_l2": 0.010696217401613108,
+      "max_absolute_error": 0.039559245109558105
+    },
+    "v_new_2": {
+      "finite": true,
+      "relative_l2": 0.014200420148717405,
+      "max_absolute_error": 0.024263650178909302
+    },
+    "k_new_3": {
+      "finite": true,
+      "relative_l2": 0.015565977193170684,
+      "max_absolute_error": 0.049951791763305664
+    },
+    "v_new_3": {
+      "finite": true,
+      "relative_l2": 0.021583585842292984,
+      "max_absolute_error": 0.02052938938140869
+    },
+    "k_new_4": {
+      "finite": true,
+      "relative_l2": 0.016605600439582216,
+      "max_absolute_error": 0.07951343059539795
+    },
+    "v_new_4": {
+      "finite": true,
+      "relative_l2": 0.02245305689448285,
+      "max_absolute_error": 0.031216859817504883
+    },
+    "k_new_5": {
+      "finite": true,
+      "relative_l2": 0.017881703357475517,
+      "max_absolute_error": 0.06754612922668457
+    },
+    "v_new_5": {
+      "finite": true,
+      "relative_l2": 0.027063196773078044,
+      "max_absolute_error": 0.05539083480834961
+    },
+    "k_new_6": {
+      "finite": true,
+      "relative_l2": 0.01842240444322585,
+      "max_absolute_error": 0.07516896724700928
+    },
+    "v_new_6": {
+      "finite": true,
+      "relative_l2": 0.026067491548697565,
+      "max_absolute_error": 0.05644726753234863
+    },
+    "k_new_7": {
+      "finite": true,
+      "relative_l2": 0.018142382614376062,
+      "max_absolute_error": 0.12753915786743164
+    },
+    "v_new_7": {
+      "finite": true,
+      "relative_l2": 0.02789881933191615,
+      "max_absolute_error": 0.08940106630325317
+    },
+    "k_new_8": {
+      "finite": true,
+      "relative_l2": 0.03538753110125184,
+      "max_absolute_error": 0.3237912654876709
+    },
+    "v_new_8": {
+      "finite": true,
+      "relative_l2": 0.031757059403891355,
+      "max_absolute_error": 0.12621372938156128
+    },
+    "k_new_9": {
+      "finite": true,
+      "relative_l2": 0.02946630212389877,
+      "max_absolute_error": 0.20687294006347656
+    },
+    "v_new_9": {
+      "finite": true,
+      "relative_l2": 0.0410935220017059,
+      "max_absolute_error": 0.10105907917022705
+    },
+    "k_new_10": {
+      "finite": true,
+      "relative_l2": 0.03868878223648271,
+      "max_absolute_error": 0.29683971405029297
+    },
+    "v_new_10": {
+      "finite": true,
+      "relative_l2": 0.04353096853260796,
+      "max_absolute_error": 0.15246811509132385
+    },
+    "k_new_11": {
+      "finite": true,
+      "relative_l2": 0.025425861330482268,
+      "max_absolute_error": 0.06829637289047241
+    },
+    "v_new_11": {
+      "finite": true,
+      "relative_l2": 0.038560645325637466,
+      "max_absolute_error": 0.05334547162055969
+    },
+    "k_new_12": {
+      "finite": true,
+      "relative_l2": 0.030940147096213254,
+      "max_absolute_error": 0.07081210613250732
+    },
+    "v_new_12": {
+      "finite": true,
+      "relative_l2": 0.040426706203836246,
+      "max_absolute_error": 0.05830147862434387
+    },
+    "k_new_13": {
+      "finite": true,
+      "relative_l2": 0.026519461206918945,
+      "max_absolute_error": 0.048555076122283936
+    },
+    "v_new_13": {
+      "finite": true,
+      "relative_l2": 0.03878078958675731,
+      "max_absolute_error": 0.04993720352649689
+    },
+    "k_new_14": {
+      "finite": true,
+      "relative_l2": 0.028585934273885952,
+      "max_absolute_error": 0.05974900722503662
+    },
+    "v_new_14": {
+      "finite": true,
+      "relative_l2": 0.03967480994030925,
+      "max_absolute_error": 0.04307170212268829
+    },
+    "k_new_15": {
+      "finite": true,
+      "relative_l2": 0.02993897036692302,
+      "max_absolute_error": 0.06578877568244934
+    },
+    "v_new_15": {
+      "finite": true,
+      "relative_l2": 0.039884867539125166,
+      "max_absolute_error": 0.05992799997329712
+    },
+    "k_new_16": {
+      "finite": true,
+      "relative_l2": 0.025222216713299887,
+      "max_absolute_error": 0.06810581684112549
+    },
+    "v_new_16": {
+      "finite": true,
+      "relative_l2": 0.04295661544654563,
+      "max_absolute_error": 0.06426090747117996
+    },
+    "k_new_17": {
+      "finite": true,
+      "relative_l2": 0.02518622685558487,
+      "max_absolute_error": 0.05489437282085419
+    },
+    "v_new_17": {
+      "finite": true,
+      "relative_l2": 0.04166885321923837,
+      "max_absolute_error": 0.07821768522262573
+    },
+    "k_new_18": {
+      "finite": true,
+      "relative_l2": 0.02249964519280874,
+      "max_absolute_error": 0.07376599311828613
+    },
+    "v_new_18": {
+      "finite": true,
+      "relative_l2": 0.03234917619905517,
+      "max_absolute_error": 0.06401261687278748
+    },
+    "k_new_19": {
+      "finite": true,
+      "relative_l2": 0.024417408811186758,
+      "max_absolute_error": 0.050400495529174805
+    },
+    "v_new_19": {
+      "finite": true,
+      "relative_l2": 0.030417240097335548,
+      "max_absolute_error": 0.10018950700759888
+    }
+  },
+  "vitisai_top1": 1867,
+  "vitisai_cache_max_relative_l2": 0.04353096853260796,
+  "profile_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260924\\evidence\\minicpmo_amd_npu_speech_head\\npu_profile_2026-09-24_20-20-16_148.json",
+  "node_providers": {
+    "vitisai": 1
+  }
+}

benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/npu_profile_2026-09-24_20-20-16_148.json

@@ -0,0 +1,7 @@
+[
+{"cat" : "Session","pid" :15612,"tid" :60812,"dur" :4168307,"ts" :15,"ph" : "X","name" :"model_loading_uri","args" : {}},
+{"cat" : "Session","pid" :15612,"tid" :60812,"dur" :416170683,"ts" :4169344,"ph" : "X","name" :"session_initialization","args" : {}},
+{"cat" : "Node","pid" :15612,"tid" :60812,"dur" :38908,"ts" :420346811,"ph" : "X","name" :"vitisai_vitisai_13419061193165137375_0_0_kernel_time","args" : {"mem_bytes_requested_in_use" : "149128","output_size" : "149128","input_type_shape" : [{"float":[1,1,768]},{"float":[1,1,1,64]},{"float":[1,12,256,64]},{"float":[1,1,1,64]},{"float":[1,12,256,64]},{"float":[1,1,1,257]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]},{"float":[1,12,256,64]}],"mem_in_use_peak" : "149248","thread_scheduling_stats" : {"main_thread": {"thread_pool_name": "session-2-intra-op", "thread_id": "60812", "block_size": [], "core": -1, "Distribution": 0, "DistributionEnqueue": 0, "Run": 0, "Wait": 0, "WaitRevoke": 0}, "sub_threads": {"64968": {"num_run": 0, "core": -1},"29568": {"num_run": 0, "core": -1},"13708": {"num_run": 0, "core": -1},"65328": {"num_run": 0, "core": -1},"33412": {"num_run": 0, "core": -1},"57132": {"num_run": 0, "core": -1},"31128": {"num_run": 0, "core": -1},"39152": {"num_run": 0, "core": -1},"39708": {"num_run": 0, "core": -1},"55708": {"num_run": 0, "core": -1},"52944": {"num_run": 0, "core": -1}}},"mem_in_use_delta" : "149248","op_name" : "vitisai_13419061193165137375_0","activation_size" : "31461892","mem_arena_held_delta" : "1048576","mem_requested_in_use_delta" : "149128","node_index" : "1049","mem_bytes_in_use" : "149248","mem_arena_held" : "1048576","output_type_shape" : [{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,12,1,64]},{"float":[1,6562]}],"provider" : "vitisai","parameter_size" : "0"}},
+{"cat" : "Session","pid" :15612,"tid" :60812,"dur" :38934,"ts" :420346805,"ph" : "X","name" :"SequentialExecutor::Execute","args" : {}},
+{"cat" : "Session","pid" :15612,"tid" :60812,"dur" :39016,"ts" :420346752,"ph" : "X","name" :"model_run","args" : {}}
+]

benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md

@@ -59,7 +59,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
 | PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | scoped E2E: Omni RTX Vulkan talker/codec + CPU predictor complete WAV and restart (separate CUDA stream has playback jitter) | scoped E2E: Omni FP8 Triton text+image on RTX; separate NVFP4 Windows startup failed | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
 | PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni Radeon talker/codec + CPU predictor; fresh restart after in-flight cancel | scoped E2E: Omni Q4_K_M GGUF Radeon text+image (20 each; slower than CPU here) | scoped E2E: Omni Radeon+CPU image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU+Radeon Cosmos (WSL/native Windows) |
-| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E (NPU pre-decoder boundary 2.27% L2, NPU-prefix+CPU-decoder waveform 9.69% L2; full mixed short window 15.7% L2) | NOT E2E | NOT E2E | NOT E2E |
+| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E (NPU pre-decoder boundary 2.27% L2, NPU-prefix+CPU-decoder waveform 9.69% L2; full mixed short window 15.7% L2) | NOT E2E | NOT E2E (speech-head step on NPU; token matched, logits/cache numeric failed) | NOT E2E |
 | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
 | PC Snapdragon X Elite CRD / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E (CPU talker/vocoder steps; two-frame QNN vocoder on NPU, component RTF 22.89) | NOT E2E | NOT E2E (NPU speech-head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
 | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | NOT E2E (one Spark attention layer on NPU) | NOT E2E (FP16 talker token mismatch; CPU talker + vocoder profiled; 25-frame and two-frame QNN vocoder memory failures; historical 25-frame TFLite mixed vocoder component RTF 0.420; pinned 25-frame mixed CPU/GPU vocoder profiled at RTF 0.335; generated-code waveform measured; short chunk RTF 3.49) | NOT E2E | NOT E2E (NPU speech head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
@@ -99,7 +99,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 | PC HX370 + AMD NPU / Windows worker | Spark-X2.5 | COMPONENT_PASS; NOT_E2E_PROFILEABLE | The real 1.7B output head (RMS norm and full 131,072-logit projection) ran through Omni `external.graph.v1` with VitisAI: one fused NPU partition and seven CPU nodes per request. The A16W8 graph matched top-1 on four captured activations (full-logit SNR 55.43–56.81 dB). One warmup plus 20 serial repeated-input component requests gave p50/p95 10.21/12.44 ms, versus 17.97/20.29 ms for the original FP32 output head on the same Omni CPU stage. NPU startup was 18.21 s versus 4.48 s for FP32 CPU, and NPU worker peak RSS was 3.52 GiB versus 2.06 GiB. No full prefill/decode/KV/sampling, real handoff, power, or sustained run was verified. | Measure an actual vLLM-to-output-head handoff and whole-generation cost, state/cancellation, memory admission and quality before considering a split plan; [component evidence](../../../e2e_expansion_20260923/evidence/spark_amd_npu_output_head/README.md). |
 | PC HX370 + AMD NPU / Windows worker | Qwen3-TTS 0.6B CustomVoice | PREFIX_NPU_COMPONENT_PASS; MIXED_NPU_CPU_WAVEFORM_NUMERIC_FAILED; NOT_E2E_PROFILEABLE | The original real-weight FP32 25-frame and pinned two-frame Code2Wav graphs reproduced retained CPU waveforms, but VitisAI EP 1.8.63.0 crashed during session creation. Four extracted two-frame prefixes (18/50/97/192 nodes) then each executed one actual VitisAI NPU partition, with one-fixture intermediate relative L2 0.958%/1.059%/0.391%/1.083% versus CPU; these are not waveform or task-quality passes. An isolated original 1D decoder ConvTranspose reproduced the native LLVM ArrayRef assertion. Expressing it as an equivalent 2D convolution compiled and ran on one NPU node; its synthetic-input output differed 1.636% from CPU. Rewriting all six decoder upsampling convolutions made the full two-frame CPU waveform bitwise identical to the source. The rewritten short graph then completed with two VitisAI and 139 CPU node events, but the one-fixture waveform missed the 1% gate at 15.696% relative L2 / 16.08 dB SNR. Its 2515 s session build and one cold 2.205 s inference for 0.16 s audio are component observations, not a steady profile. At the real-weight boundary before the decoder, one NPU partition produced a tensor 2.265% from CPU; routing that NPU tensor through the unchanged CPU decoder yielded a waveform 9.689% from the CPU-boundary control, versus 1.05e-6 for CPU boundary plus CPU decoder. A documented unvectorized-layout control instead gave 2.287% boundary and 10.074% CPU-decoder waveform relative L2, so that setting did not repair the route. The NPU prefix itself fails the waveform gate; complete TTS streaming, representative quality, memory and power remain unverified. The historical export revision is unattested; the two-frame source is pinned to CustomVoice revision 85e237c. | Isolate the NPU prefix precision/operation error using same-fixture waveform gates, then warm-profile a numerically qualified coarse stage and run complete Omni TTS with state, cancellation, admission and listening quality. [Raw AMD NPU bisection and rewrite evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_amd_npu_vocoder/README.md). |
 | PC HX370 + AMD NPU / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The tested A16W8 whole-vision-tower export failed NPU placement. This conclusion is specific to that export, not all future Qwen/NPU combinations. | Validate a supported export or smaller useful component, including same-checkpoint numerical quality. |
-| PC HX370 + AMD NPU / Windows worker | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
+| PC HX370 + AMD NPU / Windows worker | MiniCPM-o 4.5 | SPEECH_HEAD_NPU_COMPONENT_EXECUTED_NUMERIC_FAILED; NOT_E2E_PROFILEABLE | The real-weight fixed-cache 20-layer speech-head ONNX step from the historical export ran on HX370 native Windows with one fused VitisAI NPU node and zero CPU node events. Local ORT CPU reproduced all 41 retained outputs bitwise. NPU preserved top speech token 1867, but logits differed 2.172% relative L2 and the worst new-cache tensor 4.353% from CPU, exceeding the provisional 1% component gate. Session creation was 420.34 s; one cold step was 39.22 ms, with no repeated timing, power, full speech loop or complete multimodal request. The exact checkpoint revision is unattested; model/fixture/reference hashes and driver are pinned. | Try a same-checkpoint numerically validated export or representative A16W8 calibration, then verify continuous cache/state, complete Omni model behavior, handoff, admission, cancellation, quality and warm performance; [raw speech-head AMD NPU evidence](../../../e2e_expansion_20260924/evidence/minicpmo_amd_npu_speech_head/README.md). |
 | PC HX370 + AMD NPU / Windows worker | InternVLA-A1 | COMPONENT_EXPORT_CPU_PASS; NPU_ARTIFACTS_REJECTED; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos CI8x8 encoder exported to fixed FP32 [6,3,256,256] -> [6,16,32,32] ONNX; ORT CPU matched the source on two fixtures with relative L2 8.74e-7/2.29e-6. A16W8 Conv+MatMul QDQ changed encoder output by 11.77%/5.03% on CPU and the VitisAI compiler emitted a missing-scale fatal assertion. MatMul-only A16W8 retained near-source CPU parity, but its original and ORT-preprocessed graphs each hit a native SmallVector compiler assertion. No VitisAI node executed, so this is not NPU component placement or complete InternVLA policy support. Failures are specific to these artifacts and EP 1.8.63.0. | Produce a VitisAI-supported whole-encoder or coarse substage artifact with representative calibration and numerical/policy tolerances; then verify actual NPU nodes, same-device handoff, full-policy behavior and memory/power before changing this disposition. [raw export and failure evidence](../../../e2e_expansion_20260923/evidence/internvla_cosmos_amd_npu/README.md). |
 | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | Spark-X2.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
 | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |