Record AMD NPU Qwen3-TTS vocoder compiler failures

2026-09-24 18:29:48 +08:00
1 parent 17b737914b
commit df8f02376d
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Check actual AMD NPU placement and waveform parity for one Qwen3-TTS vocoder window."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import time
+from pathlib import Path
+
+import numpy as np
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as source:
+        for block in iter(lambda: source.read(8 << 20), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
+    reference = np.asarray(reference, dtype=np.float64)
+    actual = np.asarray(actual, dtype=np.float64)
+    if reference.shape != actual.shape:
+        raise ValueError(f"waveform shape changed: {reference.shape} versus {actual.shape}")
+    error = actual - reference
+    denom = max(float(np.linalg.norm(reference)), 1e-12)
+    relative_l2 = float(np.linalg.norm(error) / denom)
+    return {
+        "finite": bool(np.isfinite(actual).all()),
+        "relative_l2": relative_l2,
+        "snr_db": float(-20 * np.log10(max(relative_l2, 1e-12))),
+        "maximum_absolute_error": float(np.max(np.abs(error))),
+        "saturation_fraction": float(np.mean(np.abs(actual) >= 0.999)),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ("model", "fixture", "reference", "ep-dir", "report", "profile-prefix"):
+        parser.add_argument(f"--{name}", type=Path, required=True)
+    parser.add_argument("--expected-model-sha256", required=True)
+    parser.add_argument("--expected-fixture-sha256", required=True)
+    parser.add_argument("--expected-reference-sha256", required=True)
+    parser.add_argument("--context-frames", type=int, default=97)
+    parser.add_argument("--output-samples", type=int, default=48000)
+    parser.add_argument("--cpu-only", action="store_true")
+    args = parser.parse_args()
+    import onnxruntime as ort
+
+    manifest_hashes = {
+        "model": args.expected_model_sha256,
+        "fixture": args.expected_fixture_sha256,
+        "reference": args.expected_reference_sha256,
+    }
+    report = {
+        "scope": "one real-weight Qwen3-TTS Code2Wav component window; not a complete TTS stream",
+        "os": platform.platform(),
+        "onnxruntime_version": ort.__version__,
+        "input_shape": [1, 512, args.context_frames],
+        "output_shape": [1, args.output_samples],
+        "model_file": str(args.model.resolve()),
+        "fixture_file": str(args.fixture.resolve()),
+        "reference_file": str(args.reference.resolve()),
+    }
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    args.profile_prefix.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        report["artifact_sha256"] = {
+            "model": sha256(args.model),
+            "fixture": sha256(args.fixture),
+            "reference": sha256(args.reference),
+        }
+        for name, expected in manifest_hashes.items():
+            if report["artifact_sha256"][name] != expected:
+                raise ValueError(f"{name} differs from pinned historical bundle")
+        with np.load(args.fixture, allow_pickle=False) as source:
+            if source.files != ["quantized"]:
+                raise ValueError("vocoder fixture schema changed")
+            quantized = np.ascontiguousarray(source["quantized"])
+        with np.load(args.reference, allow_pickle=False) as source:
+            if source.files != ["wav"]:
+                raise ValueError("CPU reference schema changed")
+            reference = np.asarray(source["wav"])
+        if quantized.shape != (1, 512, args.context_frames) or quantized.dtype != np.float32:
+            raise ValueError("vocoder input shape or dtype changed")
+        if reference.shape != (1, args.output_samples) or reference.dtype != np.float32:
+            raise ValueError("vocoder reference shape or dtype changed")
+
+        cpu_started = time.perf_counter()
+        cpu = ort.InferenceSession(str(args.model), providers=["CPUExecutionProvider"])
+        report["cpu_load_s"] = time.perf_counter() - cpu_started
+        inputs = cpu.get_inputs()
+        if len(inputs) != 1 or inputs[0].shape != [1, 512, args.context_frames]:
+            raise ValueError("ONNX vocoder input contract changed")
+        input_name = inputs[0].name
+        cpu_started = time.perf_counter()
+        cpu_wave = cpu.run(None, {input_name: quantized})[0]
+        report["cpu_inference_s"] = time.perf_counter() - cpu_started
+        report["cpu_vs_retained_reference"] = compare(reference, cpu_wave)
+        if (not report["cpu_vs_retained_reference"]["finite"]
+                or report["cpu_vs_retained_reference"]["relative_l2"] > 1e-4):
+            raise ValueError("current ONNX CPU output differs from the retained reference")
+        del cpu
+        if args.cpu_only:
+            report["status"] = "cpu_reference_pass"
+            return
+
+        ep_dir = args.ep_dir.resolve(strict=True)
+        os.environ["PATH"] = str(ep_dir) + os.pathsep + os.environ.get("PATH", "")
+        if hasattr(os, "add_dll_directory"):
+            os.add_dll_directory(str(ep_dir))
+        ort.register_execution_provider_library("vitisai", str(ep_dir / "onnxruntime_vitisai_ep.dll"))
+        devices = [device for device in ort.get_ep_devices()
+                   if device.ep_name == "vitisai" and str(device.device.type).endswith("NPU")]
+        report["vitisai_npu_device_count"] = len(devices)
+        if not devices:
+            raise RuntimeError("VitisAI EP did not expose an NPU device")
+        options = ort.SessionOptions()
+        options.add_provider_for_devices(devices, {})
+        options.enable_profiling = True
+        options.profile_file_prefix = str(args.profile_prefix)
+        npu_started = time.perf_counter()
+        npu = ort.InferenceSession(str(args.model), sess_options=options)
+        report["npu_load_s"] = time.perf_counter() - npu_started
+        npu_started = time.perf_counter()
+        npu_wave = npu.run(None, {input_name: quantized})[0]
+        report["npu_inference_s"] = time.perf_counter() - npu_started
+        report["npu_vs_cpu"] = compare(cpu_wave, npu_wave)
+        report["npu_vs_retained_reference"] = compare(reference, npu_wave)
+        profile_path = Path(npu.end_profiling())
+        report["profile_file"] = str(profile_path)
+        profile = json.loads(profile_path.read_text(encoding="utf-8"))
+        providers = [(event.get("args") or {}).get("provider") for event in profile
+                     if event.get("cat") == "Node"]
+        report["placement_events"] = {
+            "vitisai": providers.count("vitisai"),
+            "cpu": providers.count("CPUExecutionProvider"),
+        }
+        report["status"] = (
+            "component_npu_numeric_pass"
+            if (report["placement_events"]["vitisai"] > 0
+                and report["npu_vs_cpu"]["finite"]
+                and report["npu_vs_cpu"]["relative_l2"] <= 0.01
+                and report["npu_vs_cpu"]["saturation_fraction"] < 0.01)
+            else "component_not_qualified"
+        )
+    except Exception as exc:
+        report["status"] = "failed"
+        report["error"] = f"{type(exc).__name__}: {exc}"
+        raise
+    finally:
+        args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+        print(json.dumps({"status": report["status"], "error": report.get("error"),
+                          "placement_events": report.get("placement_events")}))
+
+
+if __name__ == "__main__":
+    main()
@@ -117,6 +117,8 @@ attributed all 728 rows to NPU, but p50/p95 was **3.662/3.676 s** for
 0.16 s audio (median component RTF **22.89**). These are component-only
 results and the short NPU vocoder is too slow for playable streaming.

+A separate [HX370 AMD NPU Code2Wav attempt](evidence/qwen_tts_amd_npu_vocoder/README.md) tested the real-weight 25-frame and pinned two-frame FP32 ONNX exports on native Windows. Both matched retained ORT CPU waveforms within 1.05e-6 relative L2, but AMD VitisAI EP 1.8.63.0 aborted during graph compilation with an LLVM ArrayRef assertion before any NPU node or waveform ran. The cell remains NOT E2E; this failure is specific to the tested graphs and provider build.
+
 The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU-requested inference produces a saturated waveform, while the same binary's CPU-requested output on the exact SA device matches ONNX CPU within 4.13e-6 relative L2. Its 100-sample CPU-requested component profile had p50/p95 5.384/5.534 s with all 738 execution-detail rows on CPU. A FP32-preserving GPU-option control still saturated 84.89% of samples, so that option did not repair the tested path. RB3 GPU-requested execution produces an unsaturated but 15.73%-divergent waveform; its fresh 100-sample mixed CPU/GPU component p50/p95 was 3.099/3.133 s, not a usable TTS timing claim. On the exact RB3 device, the same binary requested on CPU matched original ONNX CPU within 4.13e-6 relative L2 on the retained fixture; its 47-sample component profile measured p50/p95 12.727/13.424 s with all 738 execution-detail rows on CPU. This is a major streaming-latency blocker. A single-fixture-calibrated RB3 INT8 QNN vocoder returned a finite but 66.93%-relative-L2 divergent waveform (3.49 dB SNR) versus ONNX CPU; its 100-sample component profile was p50/p95 1.971/1.992 s with all 728 execution-detail rows on NPU; these timings describe numerically unqualified audio. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. Corrected X Elite DirectML inference and profile both failed during provider initialization; separate FP16 QNN DLC inference returned an unsaturated waveform at 1.138% relative L2 / 38.88 dB SNR against original ONNX CPU; its placement profile timed out during QNN graph preparation without samples. A separate SA8775P FP16 QNN vocoder compile passed, but same-fixture NPU-requested inference failed after compilation without a waveform or placement samples. Fresh S24 and S25 CPU-requested TFLite vocoder inferences matched ONNX CPU within 4.04e-6 relative L2 on the retained fixture. S25 measured 100 component samples at p50/p95 4.418/4.641 s with all 738 execution-detail rows on CPU; S24 measured 100 at 4.826/4.958 s with all 738 rows on CPU. Separate explicit INT8 and W8A16 QDQ ONNX exports checked on local ORT CPU failed the same-fixture numerical gate at 246.97% and 331.83% waveform relative L2, respectively. S25 CPU talker inference preserved token 80 and its 100-sample component profile measured p50/p95 30.161/31.301 ms with all 1,800 rows on CPU. No complete on-device TTS stream or model-quality qualification is claimed.

 A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary matched the original ONNX CPU waveform within 3.45e-6 relative L2. The SA saturation is therefore not universal to the converted artifact; the exact SA CPU control passed numerical and placement checks, while the faulty GPU delegate operation remains unidentified.
@@ -0,0 +1,54 @@
+# Qwen3-TTS Code2Wav on the HX370 AMD NPU (2026-09-24)
+
+**Disposition: component NPU compile failed; NOT E2E.** On native Windows
+11 build 26200 with ONNX Runtime 1.30.0 and AMD VitisAI EP 1.8.63.0,
+both fixed-shape, real-weight FP32 Code2Wav graphs passed a separate
+same-file CPU reference check. Registering the AMD NPU and creating the
+VitisAI session then terminated the process in `vaiml.dll` with an LLVM
+`ArrayRef` assertion (`Index < Length`, line 260). Both process exit codes
+were `0xC000001D` (signed `-1073741795`). The emitted profile files
+are zero bytes. No NPU inference, node placement, waveform, latency, or
+full Qwen3-TTS stream can be claimed from these attempts.
+
+| Pinned graph | Input → output | ORT CPU versus retained reference | VitisAI result |
+|---|---|---:|---|
+| Historical 25-frame export, SHA-256 `7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b` | `[1,512,97]` → `[1,48000]` | relative L2 `6.46e-7`; unsaturated | Native compiler assertion |
+| Pinned two-frame export at revision `85e237c12c027371202489a0ec509ded67b5e4b5`, SHA-256 `e563541bf77200ac4c4ca9d6e01cd7e25c09f9274233a8d82eeccc57580830d2` | `[1,512,74]` → `[1,3840]` | relative L2 `1.05e-6`; unsaturated | Same native compiler assertion |
+
+The historical export's exact checkpoint revision remains unattested;
+its file hash, fixture and retained CPU output are pinned in
+[audit_report.json](audit_report.json). The two-frame graph derives from
+the pinned CustomVoice snapshot described in the
+[export report](../qwen_tts_vocoder_qualcomm/s25/qnn_short_chunk/export_report.json).
+The FP32 ONNX files (about 424 MB each) and historical fixture are stored
+outside Git under `/home/zhout/project/edge_infer/models/`; the two-frame
+fixture and retained reference are in the linked S25 evidence directory.
+No precision reduction, model substitution, or CPU fallback is counted as
+AMD NPU execution.
+
+The exact CPU controls are [25-frame](fp32_cpu_control.json) and
+[two-frame](short_cpu_control.json). The raw mixed-encoding native logs are
+[25-frame](fp32_attempt1.driver.log) and
+[two-frame](short_attempt1.driver.log); they contain the assertion and
+`vaiml.dll` compiler stack. Each zero-byte profile is retained to show
+that ORT did not emit NPU placement events. These are single-fixture
+component attempts with no warmup, repeated samples, power control, or
+task-quality validation.
+
+To reproduce, run
+[probe_qwen_tts_amd_npu_vocoder.py](../../../../experiments/probe_qwen_tts_amd_npu_vocoder.py)
+with the model, fixture and reference paths from the CPU reports,
+`--expected-model-sha256`, `--expected-fixture-sha256` and
+`--expected-reference-sha256` from the audit report, plus
+`--context-frames`, `--output-samples`, `--report`,
+`--profile-prefix` and `--ep-dir` pointing to the installed VitisAI
+ExecutionProvider. `--cpu-only` reproduces the parity gate without
+starting the NPU compiler. Run the NPU case in a separate process because
+the native assertion cannot be caught by Python.
+
+Next: bisect the specific ONNX operator/subgraph that trips this VitisAI
+compiler, export a version it accepts without losing same-checkpoint
+waveform quality, verify nonzero NPU node placement and measured coarse
+stage benefit, then connect talker, codec, state, admission, cancellation,
+and complete-stream quality gates through Omni. This compiler failure
+constrains these graphs and EP version only.
@@ -0,0 +1,68 @@
+{
+  "date": "2026-09-24",
+  "host": "HX370 laptop, native Windows 11 build 26200",
+  "device": "integrated AMD NPU",
+  "backend": "WindowsWorkload.EP.AMD.VitisAI.Framework.1.8 1.8.63.0, ONNX Runtime 1.30.0",
+  "precision": "FP32 ONNX graph; no graph quantization or alternate precision was silently applied",
+  "workload": "single Code2Wav component window; fixed fixture, batch 1, no concurrency",
+  "power_condition": "uncontrolled; no power measurement",
+  "status": "COMPONENT_NPU_COMPILE_FAILED_NOT_E2E",
+  "attempts": [
+    {
+      "name": "fp32",
+      "checkpoint_revision": "historical export; revision not attested",
+      "onnx_sha256": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b",
+      "fixture_sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d",
+      "reference_sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1",
+      "input_shape": [
+        1,
+        512,
+        97
+      ],
+      "output_shape": [
+        1,
+        48000
+      ],
+      "cpu_control_file": "fp32_cpu_control.json",
+      "cpu_reference_relative_l2": 6.457128070061599e-07,
+      "cpu_reference_saturation_fraction": 0.0,
+      "vitisai_process_exit_code_signed": -1073741795,
+      "vitisai_process_exit_code_hex": "0xC000001D",
+      "vitisai_log_file": "fp32_attempt1.driver.log",
+      "vitisai_log_sha256": "dcda9ae8a016e2d0bc16da81903e199fae5dd1a881d8019dc42294225f53b6de",
+      "vitisai_profile_file": "fp32_attempt1_profile_2026-09-24_18-18-30_731.json",
+      "vitisai_profile_bytes": 0,
+      "npu_inference_completed": false,
+      "npu_node_placement_verified": false,
+      "failure": "Native vaiml.dll compiler assertion: LLVM ArrayRef Index < Length, line 260; process terminated before Python report."
+    },
+    {
+      "name": "short",
+      "checkpoint_revision": "85e237c12c027371202489a0ec509ded67b5e4b5",
+      "onnx_sha256": "e563541bf77200ac4c4ca9d6e01cd7e25c09f9274233a8d82eeccc57580830d2",
+      "fixture_sha256": "77aea3554c51b9daf808748257ebfc5e0496172c4d34c8267288be984d8a0f55",
+      "reference_sha256": "989307db317d77be054f684211ae0c6a2fd24d93bcd74307c597fed430ba2a37",
+      "input_shape": [
+        1,
+        512,
+        74
+      ],
+      "output_shape": [
+        1,
+        3840
+      ],
+      "cpu_control_file": "short_cpu_control.json",
+      "cpu_reference_relative_l2": 1.0450898781329582e-06,
+      "cpu_reference_saturation_fraction": 0.0,
+      "vitisai_process_exit_code_signed": -1073741795,
+      "vitisai_process_exit_code_hex": "0xC000001D",
+      "vitisai_log_file": "short_attempt1.driver.log",
+      "vitisai_log_sha256": "13bca3ea59719211a9b35319f22f280b093a07bf5d304f06cdcea36846100d9c",
+      "vitisai_profile_file": "short_attempt1_profile_2026-09-24_18-22-57_775.json",
+      "vitisai_profile_bytes": 0,
+      "npu_inference_completed": false,
+      "npu_node_placement_verified": false,
+      "failure": "Native vaiml.dll compiler assertion: LLVM ArrayRef Index < Length, line 260; process terminated before Python report."
+    }
+  ]
+}
@@ -0,0 +1,32 @@
+{
+  "scope": "one real-weight Qwen3-TTS Code2Wav component window; not a complete TTS stream",
+  "os": "Windows-11-10.0.26200-SP0",
+  "onnxruntime_version": "1.30.0",
+  "input_shape": [
+    1,
+    512,
+    97
+  ],
+  "output_shape": [
+    1,
+    48000
+  ],
+  "model_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\qwen3tts_code2wav_c25_20260924.onnx",
+  "fixture_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\qwen3tts_vocoder_fixture_20260924.npz",
+  "reference_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\qwen3tts_vocoder_ort_cpu_output_20260924.npz",
+  "artifact_sha256": {
+    "model": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b",
+    "fixture": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d",
+    "reference": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
+  },
+  "cpu_load_s": 4.244146900018677,
+  "cpu_inference_s": 2.895080899936147,
+  "cpu_vs_retained_reference": {
+    "finite": true,
+    "relative_l2": 6.457128070061599e-07,
+    "snr_db": 123.79921199506902,
+    "maximum_absolute_error": 6.854534149169922e-07,
+    "saturation_fraction": 0.0
+  },
+  "status": "cpu_reference_pass"
+}
@@ -0,0 +1,32 @@
+{
+  "scope": "one real-weight Qwen3-TTS Code2Wav component window; not a complete TTS stream",
+  "os": "Windows-11-10.0.26200-SP0",
+  "onnxruntime_version": "1.30.0",
+  "input_shape": [
+    1,
+    512,
+    74
+  ],
+  "output_shape": [
+    1,
+    3840
+  ],
+  "model_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\qwen3tts_code2wav_c2_ctx72_20260924.onnx",
+  "fixture_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260924\\evidence\\qwen_tts_vocoder_qualcomm\\s25\\qnn_short_chunk\\fixture.npz",
+  "reference_file": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260924\\evidence\\qwen_tts_vocoder_qualcomm\\s25\\qnn_short_chunk\\ort_output.npz",
+  "artifact_sha256": {
+    "model": "e563541bf77200ac4c4ca9d6e01cd7e25c09f9274233a8d82eeccc57580830d2",
+    "fixture": "77aea3554c51b9daf808748257ebfc5e0496172c4d34c8267288be984d8a0f55",
+    "reference": "989307db317d77be054f684211ae0c6a2fd24d93bcd74307c597fed430ba2a37"
+  },
+  "cpu_load_s": 4.186218200018629,
+  "cpu_inference_s": 2.304827299900353,
+  "cpu_vs_retained_reference": {
+    "finite": true,
+    "relative_l2": 1.0450898781329582e-06,
+    "snr_db": 119.61692716907409,
+    "maximum_absolute_error": 3.948807716369629e-07,
+    "saturation_fraction": 0.0
+  },
+  "status": "cpu_reference_pass"
+}
@@ -59,7 +59,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
 | PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | scoped E2E: Omni RTX Vulkan talker/codec + CPU predictor complete WAV and restart (separate CUDA stream has playback jitter) | scoped E2E: Omni FP8 Triton text+image on RTX; separate NVFP4 Windows startup failed | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
 | PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni Radeon talker/codec + CPU predictor; fresh restart after in-flight cancel | scoped E2E: Omni Q4_K_M GGUF Radeon text+image (20 each; slower than CPU here) | scoped E2E: Omni Radeon+CPU image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU+Radeon Cosmos (WSL/native Windows) |
-| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
+| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E (two real vocoder graphs passed CPU parity; VitisAI compiler crashed before NPU placement) | NOT E2E | NOT E2E | NOT E2E |
 | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
 | PC Snapdragon X Elite CRD / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E (CPU talker/vocoder steps; two-frame QNN vocoder on NPU, component RTF 22.89) | NOT E2E | NOT E2E (NPU speech-head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
 | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | NOT E2E (one Spark attention layer on NPU) | NOT E2E (FP16 talker token mismatch; CPU talker + vocoder profiled; 25-frame and two-frame QNN vocoder memory failures; historical 25-frame TFLite mixed vocoder component RTF 0.420; pinned 25-frame mixed CPU/GPU vocoder profiled at RTF 0.335; generated-code waveform measured; short chunk RTF 3.49) | NOT E2E | NOT E2E (NPU speech head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
@@ -97,7 +97,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 | PC HX370 + Radeon 890M / Windows worker | MiniCPM-o 4.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST_HYBRID; NATIVE_STORAGE_PROFILED | The pinned Q4_K_M/F16 GGUF combined synthetic spoken-question+JPEG → text+speech route passed one warmup and 20/20 serial complete requests through a bounded Omni whole-session stage with weights on native NTFS. Nearest-rank wall p50/p95 was **16.35/17.20 s**, range 14.08–17.30 s, excluding 8.75 s startup; all requests returned the red-square sentence and nonzero 2.08–2.88 s 24 kHz speech with terminal audio events. The first WAV exactly matched pinned Whisper tiny.en ASR (WER 0, one-output intelligibility proxy). All 21 successful worker logs identified Radeon 890M Vulkan0 language (37/37 layers) and vision plus CPU Token2Wav, with zero prefetch warnings; sampled RSS peaked at 15.30 GB. The 21 GiB shared-RAM reservation admitted after file-cache reclaim; an initial identical attempt was explicitly refused before model load when available RAM had fallen below reservation. Cancellation left no stale output or reservation. A sequential same-artifact one-request UNC control took 90.63 s versus 16.89 s native and logged two prefetch warnings; the previous UNC 20-request profile remains p50/p95 87.27/91.45 s for that path. Storage is strongly associated with the slowdown, but OS cache/run order are uncontrolled and the separate CPU/Radeon profiles are not a qualified iGPU speedup comparison. This is CPU+iGPU joint execution, not all-Radeon, NPU, general quality, playable streaming or sustained thermal/power qualification. | Validate real multimodal quality, playable streaming, post-cancel restart, concurrency, loading peak and sustained power/thermal behavior; capture matched OS/shared-iGPU memory traces. [Native/UNC raw evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_native_weights_io/README.md), [earlier UNC 20-request evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_radeon_profile20/README.md). |
 | PC HX370 + Radeon 890M / Windows worker | InternVLA-A1 | SYNTHETIC_POLICY_PASS_HYBRID_OMNI; NOT_TASK_E2E | The real Place_Markpen checkpoint ran in admitted Omni whole-policy stages on both WSL CPU+Radeon and native Windows CPU+Radeon. In each, the BF16 Qwen3-VL/flow action policy stayed on CPU and a pinned six-frame FP32 Cosmos encoder used Omni's external DirectML worker; the selected Radeon 890M and graph output device were verified, without internal per-op attribution. Each placement returned 20/20 identical finite `[1,50,32]` actions after one warmup. WSL hybrid complete-request p50/p95 was 3.120/3.197 s versus its separate CPU baseline 3.175/3.219 s, with action relative L2 difference 2.305%. Native Windows hybrid was 4.575/4.725 s versus its separate CPU baseline 4.878/4.902 s, with action relative L2 difference 2.4391%. Native hybrid startup was 140.31 s versus CPU 135.60 s. These are separate one-fixture serial runs, not paired speedup or task-quality proof. FP32 encoder is an explicit precision change after DirectML BF16 aborted. Both stages verified hashes, reserved 16 GiB shared host RAM, rejected insufficient demand before launch, emitted terminal actions through public `AsyncOmni.generate`, and released reservations after in-flight cancellation/shutdown. An initial WSL hybrid shutdown quarantined 16 GiB; the lifecycle fix and reruns cleared it. The older direct hybrid zero-input profile is a different experiment. | Validate real observations/reference actions, physical units/order/step time, post-cancel restart, loading peak, concurrency and sustained tail/power before deployment; [WSL Omni evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_policy/README.md), [native Windows Omni evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_hybrid/README.md), [earlier direct component evidence](../../../e2e_expansion_20260923/evidence/internvla_cosmos_radeon890m/README.md). |
 | PC HX370 + AMD NPU / Windows worker | Spark-X2.5 | COMPONENT_PASS; NOT_E2E_PROFILEABLE | The real 1.7B output head (RMS norm and full 131,072-logit projection) ran through Omni `external.graph.v1` with VitisAI: one fused NPU partition and seven CPU nodes per request. The A16W8 graph matched top-1 on four captured activations (full-logit SNR 55.43–56.81 dB). One warmup plus 20 serial repeated-input component requests gave p50/p95 10.21/12.44 ms, versus 17.97/20.29 ms for the original FP32 output head on the same Omni CPU stage. NPU startup was 18.21 s versus 4.48 s for FP32 CPU, and NPU worker peak RSS was 3.52 GiB versus 2.06 GiB. No full prefill/decode/KV/sampling, real handoff, power, or sustained run was verified. | Measure an actual vLLM-to-output-head handoff and whole-generation cost, state/cancellation, memory admission and quality before considering a split plan; [component evidence](../../../e2e_expansion_20260923/evidence/spark_amd_npu_output_head/README.md). |
-| PC HX370 + AMD NPU / Windows worker | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
+| PC HX370 + AMD NPU / Windows worker | Qwen3-TTS 0.6B CustomVoice | CPU_COMPONENT_PARITY_PASS; VITISAI_COMPILER_CRASH; NOT_E2E_PROFILEABLE | Real-weight FP32 Code2Wav 25-frame and pinned two-frame ONNX exports reproduced their retained CPU waveforms at 6.46e-7 and 1.05e-6 relative L2 on the HX370 native Windows worker. With AMD VitisAI EP 1.8.63.0, both session creations aborted in vaiml.dll (LLVM ArrayRef assertion, exit 0xC000001D); both profiles are empty. No NPU node, waveform, latency, stage handoff or full TTS stream was verified. The historical export revision is unattested; the two-frame export is pinned to CustomVoice revision 85e237c. | Bisect the graph/compiler incompatibility, re-export a same-checkpoint numerically valid coarse vocoder stage, then verify actual NPU placement and whole-stream Omni handoff, state, memory, quality and latency; [raw AMD NPU evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_amd_npu_vocoder/README.md). |
 | PC HX370 + AMD NPU / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The tested A16W8 whole-vision-tower export failed NPU placement. This conclusion is specific to that export, not all future Qwen/NPU combinations. | Validate a supported export or smaller useful component, including same-checkpoint numerical quality. |
 | PC HX370 + AMD NPU / Windows worker | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
 | PC HX370 + AMD NPU / Windows worker | InternVLA-A1 | COMPONENT_EXPORT_CPU_PASS; NPU_ARTIFACTS_REJECTED; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos CI8x8 encoder exported to fixed FP32 [6,3,256,256] -> [6,16,32,32] ONNX; ORT CPU matched the source on two fixtures with relative L2 8.74e-7/2.29e-6. A16W8 Conv+MatMul QDQ changed encoder output by 11.77%/5.03% on CPU and the VitisAI compiler emitted a missing-scale fatal assertion. MatMul-only A16W8 retained near-source CPU parity, but its original and ORT-preprocessed graphs each hit a native SmallVector compiler assertion. No VitisAI node executed, so this is not NPU component placement or complete InternVLA policy support. Failures are specific to these artifacts and EP 1.8.63.0. | Produce a VitisAI-supported whole-encoder or coarse substage artifact with representative calibration and numerical/policy tolerances; then verify actual NPU nodes, same-device handoff, full-policy behavior and memory/power before changing this disposition. [raw export and failure evidence](../../../e2e_expansion_20260923/evidence/internvla_cosmos_amd_npu/README.md). |