Add admitted InternVLA CUDA policy on WSL and native Windows

2026-09-24 03:55:29 +08:00
1 parent 60d07a8afd
commit e5c4449a8f
@@ -25,14 +25,23 @@ async def main() -> None:
    for name in ("model-dir", "cosmos-dir", "processor-dir", "python-bin", "log-file", "output-report"):
        parser.add_argument(f"--{name}", type=Path, required=True)
    parser.add_argument("--graph-file", type=Path)
-    parser.add_argument("--placement", choices=("cpu", "radeon-cosmos"), required=True)
+    parser.add_argument("--placement", choices=("cpu", "cuda", "radeon-cosmos"), required=True)
    parser.add_argument("--capacity-gib", type=int, default=30)
    parser.add_argument("--reserve-gib", type=int, default=16)
+    parser.add_argument("--vram-capacity-gib", type=int)
+    parser.add_argument("--vram-reserve-gib", type=int)
    args = parser.parse_args()
    if args.placement == "radeon-cosmos" and args.graph_file is None:
        parser.error("Radeon placement requires graph-file")
    if args.reserve_gib < 1 or args.capacity_gib < args.reserve_gib:
        parser.error("invalid explicit host-RAM budget")
+    if args.placement == "cuda" and (
+        args.vram_capacity_gib is None or args.vram_reserve_gib is None
+        or args.vram_reserve_gib < 1 or args.vram_capacity_gib < args.vram_reserve_gib
+    ):
+        parser.error("CUDA placement requires an explicit positive VRAM capacity and reservation")
+    if args.placement != "cuda" and (args.vram_capacity_gib is not None or args.vram_reserve_gib is not None):
+        parser.error("VRAM budget is only valid for CUDA placement")

    import numpy as np
    import psutil
@@ -45,6 +54,15 @@ async def main() -> None:
    host_available_bytes = psutil.virtual_memory().available
    if args.capacity_gib << 30 > host_available_bytes:
        raise RuntimeError("declared host-RAM capacity exceeds OS available RAM before load")
+    cuda_before = None
+    if args.placement == "cuda":
+        if not torch.cuda.is_available():
+            raise RuntimeError("CUDA placement requested but no CUDA device is available")
+        free_vram, total_vram = torch.cuda.mem_get_info(0)
+        cuda_before = {"free_bytes": free_vram, "total_bytes": total_vram,
+                       "device_name": torch.cuda.get_device_name(0)}
+        if args.vram_capacity_gib << 30 > free_vram:
+            raise RuntimeError("declared CUDA capacity exceeds observed free VRAM before load")

    model = args.model_dir.resolve(strict=True)
    cosmos = args.cosmos_dir.resolve(strict=True)
@@ -63,6 +81,7 @@ async def main() -> None:
        files["graph"] = graph
    backend = {
        "name": "external.internvla.policy.v1", "placement": args.placement,
+        "expected_cuda_device_name": cuda_before["device_name"] if cuda_before else None,
        "python_bin": str(python), "expected_torch": str(torch.__version__),
        "model_dir": str(model), "cosmos_dir": str(cosmos), "processor_dir": str(processor),
        "graph_file": str(graph) if graph else None,
@@ -71,10 +90,15 @@ async def main() -> None:
        "max_input_bytes": 8 << 20, "max_action_bytes": 1 << 20,
        "start_timeout_s": 180, "request_timeout_s": 60,
    }
+    budget = {"capacities": {"host_ram": args.capacity_gib << 30},
+              "demands": {"host_ram": args.reserve_gib << 30}}
+    if args.placement == "cuda":
+        budget["capacities"]["cuda:0"] = args.vram_capacity_gib << 30
+        budget["demands"]["cuda:0"] = args.vram_reserve_gib << 30
    report = {"entrypoint": "AsyncOmni.generate", "placement": args.placement,
              "pipeline": INTERNVLA_A1_WHOLE_POLICY_PIPELINE.model_type,
              "host_available_bytes_before": host_available_bytes,
-              "budget": {"capacity_gib": args.capacity_gib, "reserve_gib": args.reserve_gib}}
+              "cuda_before": cuda_before, "budget": budget}
    engine = None
    try:
        with tempfile.TemporaryDirectory(prefix="omni-internvla-") as directory:
@@ -83,8 +107,7 @@ async def main() -> None:
                "pipeline": INTERNVLA_A1_WHOLE_POLICY_PIPELINE.model_type,
                "async_chunk": False,
                "stages": [{"stage_id": 0, "backend": backend,
-                            "resource_budget": {"capacities": {"host_ram": args.capacity_gib << 30},
-                                                "demands": {"host_ram": args.reserve_gib << 30}}}],
+                            "resource_budget": budget}],
            }), encoding="utf-8")
            started = time.perf_counter()
            engine = AsyncOmni(
@@ -30,11 +30,13 @@ async def main() -> None:
    for name in ("model-dir", "cosmos-dir", "processor-dir", "python-bin", "log-file", "output-report"):
        parser.add_argument(f"--{name}", type=Path, required=True)
    parser.add_argument("--graph-file", type=Path)
-    parser.add_argument("--placement", choices=("cpu", "radeon-cosmos"), required=True)
+    parser.add_argument("--placement", choices=("cpu", "cuda", "radeon-cosmos"), required=True)
    parser.add_argument("--output-actions", type=Path)
    parser.add_argument("--reference-actions", type=Path)
    parser.add_argument("--capacity-gib", type=int, default=30)
    parser.add_argument("--reserve-gib", type=int, default=16)
+    parser.add_argument("--vram-capacity-gib", type=int)
+    parser.add_argument("--vram-reserve-gib", type=int)
    parser.add_argument("--warmups", type=int, default=1)
    parser.add_argument("--repeats", type=int, default=20)
    parser.add_argument("--admission-refusal", action="store_true")
@@ -44,6 +46,13 @@ async def main() -> None:
        parser.error("invalid profile counts or explicit memory budget")
    if args.placement == "radeon-cosmos" and args.graph_file is None:
        parser.error("Radeon placement requires --graph-file")
+    if args.placement == "cuda" and (
+        args.vram_capacity_gib is None or args.vram_reserve_gib is None
+        or args.vram_reserve_gib < 1 or args.vram_capacity_gib < args.vram_reserve_gib
+    ):
+        parser.error("CUDA placement requires an explicit positive VRAM capacity and reservation")
+    if args.placement != "cuda" and (args.vram_capacity_gib is not None or args.vram_reserve_gib is not None):
+        parser.error("VRAM budget is only valid for CUDA placement")

    import numpy as np
    import psutil
@@ -74,8 +83,19 @@ async def main() -> None:
    hashes = {name: sha256(path) for name, path in files.items()}
    import torch

+    cuda_before = None
+    if args.placement == "cuda":
+        if not torch.cuda.is_available():
+            raise RuntimeError("CUDA placement requested but no CUDA device is available")
+        free_vram, total_vram = torch.cuda.mem_get_info(0)
+        cuda_before = {"free_bytes": free_vram, "total_bytes": total_vram,
+                       "device_name": torch.cuda.get_device_name(0)}
+        if args.vram_capacity_gib << 30 > free_vram:
+            raise RuntimeError("declared CUDA capacity exceeds observed free VRAM before load")
+
    backend = {
        "name": "external.internvla.policy.v1", "placement": args.placement,
+        "expected_cuda_device_name": cuda_before["device_name"] if cuda_before else None,
        "python_bin": str(python), "expected_torch": str(torch.__version__),
        "model_dir": str(model), "cosmos_dir": str(cosmos), "processor_dir": str(processor),
        "graph_file": str(graph) if graph else None, "artifact_sha256": hashes,
@@ -85,6 +105,9 @@ async def main() -> None:
    }
    budget = {"capacities": {"host_ram": args.capacity_gib << 30},
              "demands": {"host_ram": args.reserve_gib << 30}}
+    if args.placement == "cuda":
+        budget["capacities"]["cuda:0"] = args.vram_capacity_gib << 30
+        budget["demands"]["cuda:0"] = args.vram_reserve_gib << 30
    deploy = DeployConfig(async_chunk=False, stages=[StageDeployConfig(
        stage_id=0, backend=backend, resource_budget=budget,
    )])
@@ -96,6 +119,7 @@ async def main() -> None:
        "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
        "placement": args.placement, "artifact_sha256": hashes, "budget": budget,
        "host_available_bytes_before": host_available_bytes,
+        "cuda_before": cuda_before,
        "warmups": args.warmups, "repeats": args.repeats,
    }
    try:
@@ -4,6 +4,10 @@ This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evid

 InternVLA also has a separate [admitted Omni whole-policy synthetic profile](evidence/internvla_omni_policy/README.md) on WSL CPU and WSL CPU+Radeon 890M: one warmup plus 20 serial complete requests per placement, explicit 16 GiB shared-RAM reservation, pre-launch refusal, hybrid in-flight abort, and a public `AsyncOmni.generate` action output. The two complete-request profiles are 3.175/3.219 s CPU and 3.120/3.197 s hybrid p50/p95; they use a patterned fixture distinct from the older direct hybrid run. Action relative L2 difference is 2.305% without a task tolerance. This advances integration depth only; robot-task E2E remains unqualified.

+The [WSL RTX 5090 Laptop Omni CUDA policy profile](evidence/internvla_omni_cuda/README.md) used the same real Place_Markpen checkpoint and patterned fixture with the BF16 policy and Cosmos encoder resident on CUDA. One warmup plus 20/20 serial terminal action requests had complete-request wall p50/p95 0.382/0.399 s, excluding 43.82 s startup. A 16 GiB host/16 GiB VRAM reservation admitted the route, an 8 GiB VRAM demand was refused before worker launch, and an in-flight abort cleared both ledgers. The public action entrypoint passed. Relative to the separate WSL CPU output, action relative L2 difference was 2.006% without a task tolerance. This is synthetic policy integration, not physical robot-task qualification or a comparison with the earlier zero-input direct CUDA forward.
+
+The [native Windows RTX 5090 Laptop Omni CUDA policy profile](evidence/internvla_omni_windows_cuda/README.md) separately ran the same patterned fixture and real checkpoint. One warmup plus 20/20 serial terminal action requests had complete-request wall p50/p95 0.746/0.806 s, excluding 147.77 s startup that included loading weights from a WSL UNC path. A 16 GiB host/16 GiB VRAM reservation, 8 GiB VRAM pre-launch refusal, in-flight abort, and public action entrypoint passed. Relative L2 was 2.340% versus its separate native CPU output and 1.555% versus the WSL CUDA action chunk; there is no task tolerance. These OS runs are not a paired speed comparison or robot-task qualification.
+
 A further [native Windows CPU Omni policy profile](evidence/internvla_omni_windows_cpu/README.md) ran the same patterned synthetic fixture with the real Place_Markpen checkpoint: one warmup plus 20/20 identical finite action chunks, nearest-rank complete-request wall p50/p95 4.878/4.902 s, and 135.60 s startup. The public action entrypoint, pre-launch 8 GiB refusal and in-flight abort passed with a 16 GiB host-RAM reservation and zero ledger residue. The native action chunk differed from the WSL CPU output by 2.1795% relative L2; no robot-task tolerance exists. The earlier one-forward native CPU result used a zero-input fixture and must not be compared as the same profile.

 The matching [native Windows CPU+Radeon Omni policy profile](evidence/internvla_omni_windows_hybrid/README.md) kept the BF16 action policy on CPU and ran only the pinned FP32 six-frame Cosmos encoder through a native DirectML worker. It returned 20/20 identical finite action chunks after one warmup at nearest-rank complete-request wall p50/p95 4.575/4.725 s, versus the separate native CPU baseline 4.878/4.902 s; startup was 140.31 s versus 135.60 s. Hybrid-vs-CPU action relative L2 difference was 2.4391% without a task tolerance. Native hybrid pre-launch refusal, in-flight abort and public `AsyncOmni.generate` action output passed with a 16 GiB shared-RAM reservation. The output device was verified as Radeon 890M, not every internal graph op. This is a synthetic integration pass, not robot-task E2E or sustained benefit.
@@ -14,10 +18,12 @@ A later [native Windows Radeon MiniCPM-o Omni profile](evidence/minicpmo_omni_ra
 |---|---|---|---|
 | Native Windows HX370 Radeon 890M / InternVLA-A1 Cosmos CI8x8 encoder | Component C; unchanged Omni encoder source in separate DirectML/PyTorch 2.4 environment, not an Omni full-policy stage | One warmup + 20 serial patterned 256×256 image encodes, including upload/readback: Radeon p50/p95 26.56/27.49 ms versus CPU 156.49/158.71 ms; relative L2 error 4.73e-07. Encoder weights and output device checked. | Complete policy/action and real-observation parity; policy/controller runtime compatibility, bounded handoff, shared-RAM admission and end-to-end benefit. [Raw evidence](evidence/internvla_cosmos_radeon890m/README.md). |
 | WSL HX370 CPU policy + Radeon 890M DirectML Cosmos / InternVLA-A1 Place_Markpen | Complete synthetic action-policy path through Omni diffusion and its external graph worker; FP32 encoder to BF16 policy, not robot-task qualification | One warmup pair + 20 alternating measured CPU/hybrid complete requests returned finite `[1,50,32]` actions. Hybrid wall p50/p95 2.952/3.322 s versus CPU 3.178/3.266 s; 16/20 paired faster but the hybrid tail regressed. Action relative L2 difference 0.9007%, cosine 0.999959. The selected Radeon adapter and graph hash were checked. | Real observations/reference actions, units/timestamps, integrated admission/cancellation, loading peak and sustained tail/power; [raw evidence](evidence/internvla_cosmos_radeon890m/README.md). |
+| WSL HX370 + RTX 5090 Laptop / InternVLA-A1 Place_Markpen | Admitted Omni whole-policy BF16 CUDA stage (P for one patterned synthetic action request; not robot-task E2E) | One warmup + 20/20 identical finite action chunks; complete-request p50/p95 0.382/0.399 s, startup 43.82 s. Verified CUDA policy/Cosmos placement and 7.86 GB peak PyTorch VRAM reservation; 16 GiB host plus 16 GiB VRAM admitted, 8 GiB VRAM refused before launch, public action and cancellation passed. Action relative L2 versus separate CPU output 2.006% without task tolerance. | Real observations/reference actions, physical metadata, post-cancel restart, loading peak, concurrency and sustained power/thermal behavior; [raw evidence](evidence/internvla_omni_cuda/README.md). |
+| Native Windows HX370 + RTX 5090 Laptop / InternVLA-A1 Place_Markpen | Admitted Omni whole-policy BF16 CUDA stage (P for one patterned synthetic action request; not robot-task E2E) | One warmup + 20/20 identical finite action chunks; complete-request p50/p95 0.746/0.806 s, startup 147.77 s. Verified CUDA policy/Cosmos placement and 7.86 GB peak PyTorch VRAM reservation; 16 GiB host plus 16 GiB VRAM admitted, 8 GiB VRAM refused before launch, public action and cancellation passed. Action relative L2 versus separate native CPU output 2.340% without task tolerance. | Real observations/reference actions, physical metadata, post-cancel restart, loading peak, concurrency and sustained power/thermal behavior; [raw evidence](evidence/internvla_omni_windows_cuda/README.md). |
 | WSL HX370 CPU / Qwen3-TTS 0.6B CustomVoice BF16 | Scoped complete-request Omni resident CPU whole-session stage (P for two named texts; not playable streaming) | One warmup + 20 serial requests returned identical 3.44 s WAVs; nearest-rank complete-request wall p50/p95 8.27/8.47 s (RTF 2.40/2.46), excluding 19.82 s startup. One pinned Whisper tiny.en transcription matched the reference (WER 0); public `AsyncOmni` audio, 10 GiB admission, 4 GiB refusal and cancellation drain passed. The WSL PCM differs from the native Windows output for the same checkpoint/settings. | Cross-OS numerical/quality investigation, playable streaming, post-cancel restart, loading peak, concurrency and sustained power; [raw evidence](evidence/qwen_tts_omni_wsl_cpu/README.md). |
 | WSL HX370 CPU / MiniCPM-o 4.5 Q4_K_M/F16 GGUF | Scoped complete-request Omni C++ whole-session graph stage (P for one synthetic combined spoken-question+image input; CPU-only model execution) | All 20 measured requests after one warmup answered the red-square question correctly with complete nonzero 24 kHz speech. Nearest-rank complete-request wall p50/p95 22.67/24.83 s; audio duration 1.60–2.92 s, maximum sampled worker RSS 13.70 GB. One WAV independently transcribed exactly (Whisper tiny.en WER 0). 24 GiB admission, 8 GiB refusal and cancellation with zero remaining ledger passed. | Real multimodal suites, playable streaming, speech alignment, post-cancel restart, loading peak, concurrency and sustained power; [raw evidence](evidence/minicpmo_omni_wsl_cpu/README.md). |
-| Native Windows HX370 CPU / MiniCPM-o 4.5 Q4_K_M/F16 GGUF | Scoped complete-request Omni C++ whole-session graph stage (P for one synthetic combined audio+image input) | Correct red-square answer, 2.24 s 24 kHz WAV; one cold request 24.34 s, sampled RSS 13.11 GB. 24 GiB admission, 8 GiB refusal and in-flight cancellation with zero remaining reservation passed. | Real multimodal suites, playable streaming, repeated latency, loading peak, post-cancel restart and sustained power; [raw evidence](evidence/minicpmo_omni_cpp/README.md). |
-| Native Windows HX370 CPU+Radeon 890M / same MiniCPM-o GGUF | Scoped complete-request Omni hybrid graph stage (P for one synthetic combined audio+image input) | Correct red-square answer, 2.60 s 24 kHz WAV; one cold request 18.27 s, sampled RSS 15.57 GB. Log verifies Radeon language+vision and CPU TTS/Token2Wav; cancellation drained without stale output or reservation. | Real multimodal suites, playable streaming, repeated/concurrent latency, loading peak and sustained power; [raw evidence](evidence/minicpmo_omni_cpp/README.md). |
+| Native Windows HX370 CPU / MiniCPM-o 4.5 Q4_K_M/F16 GGUF | Scoped complete-request Omni C++ whole-session graph stage (P for one synthetic combined audio+image input); unstable latency | The original cold request took 24.34 s, while a later same-fixture control took 73.63 s and logged prefetch warnings despite 32.28 GB OS-available before load. Both answered the red-square question and emitted nonzero speech. Admission, refusal and cancellation passed; there is no repeated CPU latency distribution or diagnosed cause. | Real multimodal suites, playable streaming, matched-condition latency/resource trace, loading peak, post-cancel restart and sustained power; [original evidence](evidence/minicpmo_omni_cpp/README.md), [later control](evidence/minicpmo_omni_radeon_profile20/README.md). |
+| Native Windows HX370 CPU+Radeon 890M / same MiniCPM-o GGUF | Scoped complete-request Omni hybrid graph stage (P for one synthetic combined audio+image input); unstable latency | After one warmup, 20/20 serial requests answered correctly with complete speech; complete-request wall p50/p95 87.27/91.45 s. Logs verify Radeon language+vision and CPU TTS/Token2Wav. The earlier one-cold-request smoke was 18.27 s under different conditions. Cancellation drained without stale output or reservation; the slowdown's cause remains unqualified. | Real multimodal suites, playable streaming, matched-condition CPU/Radeon resource trace, loading peak and sustained power; [20-request evidence](evidence/minicpmo_omni_radeon_profile20/README.md), [earlier smoke](evidence/minicpmo_omni_cpp/README.md). |
 | Native Windows HX370 CPU+Radeon 890M / Qwen3-TTS 0.6B CustomVoice Q8_0 GGUF plus F16 codec | Scoped complete standalone CrispASR text-to-WAV path with Vulkan talker/codec and CPU FP32 code predictor (P for two named sentences; outside Omni; **not** all-Radeon or NPU) | Two 24 kHz WAVs of 2.56 s and 3.20 s; independently pinned Whisper tiny.en ASR WER 0 on both. One warmup + 20 serial measured resident-server requests for the first prompt had client wall p50/p95 3.310/3.345 s, all PCM hashes identical; RTF 1.29/1.31. The default Vulkan setting fell back to CPU, while all-Vulkan talker+predictor hit a 360-frame runaway cap. | Explicit Omni hybrid stage binding, quality and playback gates, loading-peak admission, cancellation, broader prompts, concurrency and sustained power/thermal behavior; [joint raw evidence](evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML attempt](evidence/qwen_tts_radeon890m_directml/README.md). |
 | Native Windows HX370 CPU+Radeon 890M / same Qwen3-TTS 0.6B CustomVoice GGUFs | Scoped complete-request Omni CrispASR hybrid stage (P for two named sentences; **not** all-Radeon or NPU) | Both 24 kHz PCM outputs exactly match the earlier standalone hashes. One warmup + 20 serial Omni requests for 2.56 s of audio had 20/20 identical PCM, nearest-rank wall p50/p95 3.298/3.336 s (RTF 1.29/1.30). Public AsyncOmni audio passed; cancellation drained the worker and ledger. | Playable streaming, post-cancel restart, broad speech quality, loading/iGPU peak, concurrency and sustained real-time/thermal gates; [Omni raw evidence](evidence/qwen_tts_omni_hybrid/README.md). |
 | Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | See the subsequent Omni CPU binding below; playable streaming, speech quality, memory loading peak and sustained behavior remain open; [standalone raw evidence](evidence/qwen_tts_native_cpu/README.md). |
@@ -0,0 +1,31 @@
+# WSL RTX 5090 Laptop: admitted InternVLA Omni action stage
+
+**Scoped synthetic policy path passed; robot-task E2E remains unqualified.** The real `InternVLA-A1-3B-FT-Place_Markpen` checkpoint (revision `0a557c6a503e6545b369bff2f34c76a410bbd190`) strict-loaded through the existing Omni `InternVLAA1Pipeline` in a bounded whole-policy graph stage. Both the BF16 policy and BF16 Cosmos encoder ran on the NVIDIA GeForce RTX 5090 **Laptop** GPU; action output was copied to a terminal float32 `[1,50,32]` host buffer. This is an explicit `cuda` plan, not a CPU fallback or a GPU component summed into a pipeline estimate.
+
+The host was Ryzen AI 9 HX 370 under Ubuntu 26.04/WSL2 kernel `6.18.33.2-microsoft-standard-WSL2`, with Windows NVIDIA driver 610.71 and 24,463 MiB GPU memory. The worker used Python 3.12, PyTorch 2.13.0+cu130 and installed vLLM 0.29.0 with this Omni checkout. Checkpoint, processor, Cosmos and interpreter SHA256 values, actual device name, precision and state metadata are in the [20-request report](final_profile20.json). The input was three patterned normalized synthetic two-frame camera histories, valid masks, zero `[1,32]` state, fixed text task and zero `[1,50,32]` diffusion noise. It was **not** a real robot observation or a reference-action quality case.
+
+After one warmup, **20/20 serial complete policy requests** returned one identical finite action hash. Nearest-rank complete-request wall p50/p95 was **0.382/0.399 s** (range 0.367–0.408 s), excluding **43.82 s** controller/worker startup; each request remained in the same loaded worker. The separate [public `AsyncOmni.generate` request](final_public.json) passed with one terminal action event; its first-request wall was 0.965 s after 45.69 s startup. These numbers are specific to this fixed synthetic input, one concurrency slot and current power/system state. The earlier direct CUDA zero-observation forward is a different workload and measurement boundary.
+
+The stage reserved **16 GiB host RAM plus 16 GiB `cuda:0` VRAM** under explicit 28/21 GiB ceilings, each checked against OS/GPU availability before loading. The real policy reported 7.80 GB CUDA reserved after load; the maximum PyTorch CUDA peak reserved across measured requests was **7.86 GB**, below the VRAM reservation. Loaded worker process-tree RSS was 8.62 GB, which is not a loading-peak bound. An [8 GiB VRAM demand](refusal.json) was rejected before worker launch because pinned policy/Cosmos weights plus explicit workspace/headroom exceeded the reservation; no refusal worker log was created. The [in-flight abort](final_profile20.json) emitted no stale action and left zero host/VRAM reservation or quarantine; shutdown did likewise.
+
+Against the separate patterned WSL CPU Omni action array, CUDA's first measured chunk had maximum absolute difference **0.01890**, relative L2 **2.006%** and cosine **0.999809**. No task tolerance has been defined, so this is numerical characterization, not task equivalence. Separate [CPU](cpu_regression.json) and [CPU+Radeon](radeon_regression.json) one-request checks passed after the worker response added CUDA peak metadata (zero on both), preserving those routes' response contract. The [native Windows CUDA Omni route](../internvla_omni_windows_cuda/README.md) was subsequently profiled separately under its own host-memory condition; its timing and action output must not be substituted for this WSL run.
+
+The action metadata explicitly reports `action_mode=delta`, unverified units and joint order, unknown step time, and `control_ready=false`. Physical action quality, deadlines, real observation/reference pairs, post-cancel restart, concurrent admission, weight-loading peak, full-device GPU memory, sustained tail latency, power and thermal behavior remain open. The [driver](final_profile20_driver.log) and [worker](final_profile20_worker.log) logs, first measured [action array](final_profile20_actions.npy), [public logs](final_public_driver.log), [refusal log](refusal_driver.log) and [CPU regression log](cpu_regression_driver.log) are retained.
+
+Reproduce from this checkout with the pinned local artifacts:
+
+```bash
+ROOT=/home/zhout/project/edge_infer
+EVIDENCE=benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla_omni_cuda
+PYTHONPATH="$PWD" "$ROOT/.venvs/omni-cuda-029/bin/python" benchmarks/edge_harness/experiments/probe_omni_internvla_policy.py \
+  --model-dir "$ROOT/models/InternVLA-A1-3B-FT-Place_Markpen" \
+  --processor-dir "$ROOT/models/Qwen3-VL-2B-Instruct-processor" \
+  --cosmos-dir "$ROOT/models/Cosmos-Tokenizer-CI8x8" \
+  --python-bin "$ROOT/.venvs/omni-cuda-029/bin/python" --placement cuda \
+  --capacity-gib 28 --reserve-gib 16 --vram-capacity-gib 21 --vram-reserve-gib 16 \
+  --warmups 1 --repeats 20 --abort-check \
+  --log-file "$EVIDENCE/final_profile20_worker.log" \
+  --output-report "$EVIDENCE/final_profile20.json" \
+  --output-actions "$EVIDENCE/final_profile20_actions.npy" \
+  --reference-actions benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla_omni_policy/cpu_profile20_actions.npy
+```
@@ -0,0 +1,248 @@
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cpu",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "demands": {
+      "host_ram": 17179869184
+    }
+  },
+  "host_available_bytes_before": 31391879168,
+  "cuda_before": null,
+  "warmups": 0,
+  "repeats": 1,
+  "startup_s": 39.85692745097913,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "298079db103a45a2a982ae8938d1762a",
+    "worker_pid": 34833,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cpu",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cpu",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.394601831038017,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 9271812096,
+    "loaded_process_tree": [
+      {
+        "pid": 34833,
+        "rss_bytes": 9271812096,
+        "private_bytes": 9271812096
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [],
+  "measured": [
+    {
+      "wall_s": 3.220823811017908,
+      "action_sha256": "007f3087423df63b578193ec3f95e353613219914f145b85ca54c403e9088a25",
+      "stage_event": {
+        "request_id": "internvla-measured-0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "298079db103a45a2a982ae8938d1762a",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "298079db103a45a2a982ae8938d1762a",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      },
+      "action_metadata": {
+        "observation_timestamp_ns": 1790191799880729146,
+        "generation_timestamp_ns": 1790191803086231082,
+        "observation_age_s": 3.205501936,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "metrics": {
+        "policy_wall_s": 3.2146441329969093,
+        "worker_wall_s": 3.1897658859961666,
+        "cuda_peak_reserved_bytes": 0
+      }
+    }
+  ],
+  "nearest_rank_p50_wall_s": 3.220823811017908,
+  "nearest_rank_p95_wall_s": 3.220823811017908,
+  "unique_action_hashes": [
+    "007f3087423df63b578193ec3f95e353613219914f145b85ca54c403e9088a25"
+  ],
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "298079db103a45a2a982ae8938d1762a",
+    "worker_pid": 34833,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cpu",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cpu",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.394601831038017,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 9271812096,
+    "loaded_process_tree": [
+      {
+        "pid": 34833,
+        "rss_bytes": 9271812096,
+        "private_bytes": 9271812096
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "reserved": {
+      "host_ram": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,211 @@
+/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
+ --> vLLM version 0.28.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+W0924 03:29:14.249000 34758 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+W0924 03:29:14.270000 34758 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+INFO 09-24 03:29:15 [importing.py:53] Triton is installed but 0 active driver(s) found (expected 1). Disabling Triton to prevent runtime errors.
+WARNING 09-24 03:29:15 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
+INFO 09-24 03:29:15 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
+INFO 09-24 03:29:16 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:29:16 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:29:16 [patch.py:646] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
+INFO 09-24 03:29:16 [patch.py:756] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
+INFO 09-24 03:29:20 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:29:20 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cpu",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "demands": {
+      "host_ram": 17179869184
+    }
+  },
+  "host_available_bytes_before": 31391879168,
+  "cuda_before": null,
+  "warmups": 0,
+  "repeats": 1,
+  "startup_s": 39.85692745097913,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "298079db103a45a2a982ae8938d1762a",
+    "worker_pid": 34833,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cpu",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cpu",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.394601831038017,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 9271812096,
+    "loaded_process_tree": [
+      {
+        "pid": 34833,
+        "rss_bytes": 9271812096,
+        "private_bytes": 9271812096
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [],
+  "nearest_rank_p50_wall_s": 3.220823811017908,
+  "nearest_rank_p95_wall_s": 3.220823811017908,
+  "unique_action_hashes": [
+    "007f3087423df63b578193ec3f95e353613219914f145b85ca54c403e9088a25"
+  ],
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "298079db103a45a2a982ae8938d1762a",
+    "worker_pid": 34833,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cpu",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cpu",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.394601831038017,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 9271812096,
+    "loaded_process_tree": [
+      {
+        "pid": 34833,
+        "rss_bytes": 9271812096,
+        "private_bytes": 9271812096
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "reserved": {
+      "host_ram": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,18 @@
+/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
+ --> vLLM version 0.28.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+W0924 03:29:27.661000 34833 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+W0924 03:29:27.685000 34833 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+WARNING 09-24 03:29:28 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
+INFO 09-24 03:29:28 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
+INFO 09-24 03:29:29 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:29:29 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:29:29 [patch.py:646] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
+INFO 09-24 03:29:30 [patch.py:756] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
+INFO 09-24 03:29:30 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=52363 props={"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8", "placement": "cpu", "torch": "2.13.0+cpu", "policy_dtype": "bfloat16", "cosmos_dtype": "bfloat16", "policy_device": "cpu", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": null, "cuda_device_index": null, "cuda_allocated_after_load_bytes": null, "cuda_reserved_after_load_bytes": null, "policy_load_s": 29.394601831038017, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": null}
+127.0.0.1 - - [24/Sep/2026 03:29:59] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:29:59] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:30:03] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,283 @@
+INFO 09-24 03:30:36 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:30:36 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:30:36 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-24 03:30:40 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:30:40 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "host_available_bytes_before": 31432986624,
+  "cuda_before": {
+    "free_bytes": 24242028544,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 1,
+  "repeats": 20,
+  "startup_s": 43.82135335600469,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "1754006336b84656b5c275207486c2bf",
+    "worker_pid": 35044,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cuda",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cuda",
+      "torch": "2.13.0+cu130",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cuda",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU",
+      "cuda_device_index": 0,
+      "cuda_allocated_after_load_bytes": 6451405312,
+      "cuda_reserved_after_load_bytes": 7799308288,
+      "policy_load_s": 32.09236832300667,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 8619163648,
+    "loaded_process_tree": [
+      {
+        "pid": 35044,
+        "rss_bytes": 8619163648,
+        "private_bytes": 8619163648
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 15373460732,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [
+    {
+      "wall_s": 0.9645575869944878,
+      "action_sha256": "e977580bc094f8249e49960e75a6a22efc7d0efc6a112b1f5c3be9d09c752e94",
+      "stage_event": {
+        "request_id": "internvla-warmup-0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "1754006336b84656b5c275207486c2bf",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "1754006336b84656b5c275207486c2bf",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      },
+      "action_metadata": {
+        "observation_timestamp_ns": 1790191884689416399,
+        "generation_timestamp_ns": 1790191885648976281,
+        "observation_age_s": 0.959559882,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "metrics": {
+        "policy_wall_s": 0.9591429099673405,
+        "worker_wall_s": 0.9516573559958488,
+        "cuda_peak_reserved_bytes": 7862222848
+      }
+    }
+  ],
+  "nearest_rank_p50_wall_s": 0.3816611410002224,
+  "nearest_rank_p95_wall_s": 0.3989414210082032,
+  "unique_action_hashes": [
+    "e977580bc094f8249e49960e75a6a22efc7d0efc6a112b1f5c3be9d09c752e94"
+  ],
+  "output_actions": "benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla_omni_cuda/final_profile20_actions.npy",
+  "reference_comparison": {
+    "max_abs": 0.018902286887168884,
+    "relative_l2": 0.020064717158675194,
+    "cosine": 0.9998085498809814
+  },
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "1754006336b84656b5c275207486c2bf",
+    "worker_pid": 35044,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cuda",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "cuda",
+      "torch": "2.13.0+cu130",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cuda",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU",
+      "cuda_device_index": 0,
+      "cuda_allocated_after_load_bytes": 6451405312,
+      "cuda_reserved_after_load_bytes": 7799308288,
+      "policy_load_s": 32.09236832300667,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 8619163648,
+    "loaded_process_tree": [
+      {
+        "pid": 35044,
+        "rss_bytes": 8619163648,
+        "private_bytes": 8619163648
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 15373460732,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "abort_check": {
+    "stale_output": false,
+    "ledger": {
+      "capacities": {
+        "host_ram": 30064771072,
+        "cuda:0": 22548578304
+      },
+      "reserved": {
+        "host_ram": 0,
+        "cuda:0": 0
+      },
+      "owners": [],
+      "quarantined": []
+    }
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,28 @@
+INFO 09-24 03:30:51 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:30:51 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:30:51 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-24 03:30:52 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=46691 props={"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8", "placement": "cuda", "torch": "2.13.0+cu130", "policy_dtype": "bfloat16", "cosmos_dtype": "bfloat16", "policy_device": "cuda", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU", "cuda_device_index": 0, "cuda_allocated_after_load_bytes": 6451405312, "cuda_reserved_after_load_bytes": 7799308288, "policy_load_s": 32.09236832300667, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": null}
+127.0.0.1 - - [24/Sep/2026 03:31:24] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:24] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:25] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:26] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:26] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:26] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:27] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:27] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:27] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:28] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:28] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:29] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:29] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:29] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:30] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:30] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:31] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:31] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:31] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:32] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:32] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:32] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:31:33] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,78 @@
+{
+  "entrypoint": "AsyncOmni.generate",
+  "placement": "cuda",
+  "pipeline": "internvla_a1_external_whole",
+  "host_available_bytes_before": 30188298240,
+  "cuda_before": {
+    "free_bytes": 24242028544,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "startup_s": 45.694621764996555,
+  "request_wall_s": 0.9651285309810191,
+  "outputs": [
+    {
+      "request_id": "public-internvla-1",
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "action_sha256": "e977580bc094f8249e49960e75a6a22efc7d0efc6a112b1f5c3be9d09c752e94",
+      "finite": true,
+      "metadata": {
+        "observation_timestamp_ns": 1790191737772515406,
+        "generation_timestamp_ns": 1790191738735014566,
+        "observation_age_s": 0.96249916,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "stage_event": {
+        "request_id": "public-internvla-1-a7ccbbc0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "b11061835edd447aaf2581732b2eb932",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "b11061835edd447aaf2581732b2eb932",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      }
+    }
+  ],
+  "status": "passed"
+}
@@ -0,0 +1,97 @@
+INFO 09-24 03:28:07 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:28:07 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:28:07 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-24 03:28:12 [omni_base.py:216] [AsyncOmni] Initializing with model /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+INFO 09-24 03:28:12 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+ERROR 09-24 03:28:12 [repo_utils.py:119] Error parsing config for /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen: Unrecognized model in /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen. Should have a `model_type` key in its config.json., retrying 1 of 2
+ERROR 09-24 03:28:14 [repo_utils.py:117] Error parsing config for /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen: Unrecognized model in /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen. Should have a `model_type` key in its config.json.
+INFO 09-24 03:28:14 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
+INFO 09-24 03:28:14 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+INFO 09-24 03:28:14 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
+INFO 09-24 03:28:57 [orchestrator.py:604] [Orchestrator] Starting event loop
+INFO 09-24 03:28:57 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
+INFO 09-24 03:28:57 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 45.69 seconds
+INFO 09-24 03:28:57 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+INFO 09-24 03:28:58 [omni_base.py:781] [AsyncOmni] Shutting down
+INFO 09-24 03:28:58 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
+INFO 09-24 03:28:58 [orchestrator.py:705] [Orchestrator] Received shutdown signal
+INFO 09-24 03:28:58 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
+INFO 09-24 03:28:59 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
+{
+  "entrypoint": "AsyncOmni.generate",
+  "placement": "cuda",
+  "pipeline": "internvla_a1_external_whole",
+  "host_available_bytes_before": 30188298240,
+  "cuda_before": {
+    "free_bytes": 24242028544,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "startup_s": 45.694621764996555,
+  "request_wall_s": 0.9651285309810191,
+  "outputs": [
+    {
+      "request_id": "public-internvla-1",
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "action_sha256": "e977580bc094f8249e49960e75a6a22efc7d0efc6a112b1f5c3be9d09c752e94",
+      "finite": true,
+      "metadata": {
+        "observation_timestamp_ns": 1790191737772515406,
+        "generation_timestamp_ns": 1790191738735014566,
+        "observation_age_s": 0.96249916,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "stage_event": {
+        "request_id": "public-internvla-1-a7ccbbc0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "b11061835edd447aaf2581732b2eb932",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "b11061835edd447aaf2581732b2eb932",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      }
+    }
+  ],
+  "status": "passed"
+}
@@ -0,0 +1,8 @@
+INFO 09-24 03:28:24 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:28:24 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:28:24 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-24 03:28:25 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=51567 props={"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8", "placement": "cuda", "torch": "2.13.0+cu130", "policy_dtype": "bfloat16", "cosmos_dtype": "bfloat16", "policy_device": "cuda", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU", "cuda_device_index": 0, "cuda_allocated_after_load_bytes": 6451405312, "cuda_reserved_after_load_bytes": 7799308288, "policy_load_s": 31.97907928202767, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": null}
+127.0.0.1 - - [24/Sep/2026 03:28:57] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:28:57] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:28:58] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,359 @@
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "radeon-cosmos",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+    "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "demands": {
+      "host_ram": 17179869184
+    }
+  },
+  "host_available_bytes_before": 31499673600,
+  "cuda_before": null,
+  "warmups": 0,
+  "repeats": 1,
+  "startup_s": 44.943421232979745,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "352c114f2b494f9384b2d561519db724",
+    "worker_pid": 1247789,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+      "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+    },
+    "placement": "radeon-cosmos",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "radeon-cosmos",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "float32",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.232657331973314,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": {
+        "ep": "dml",
+        "graph_path": "/home/zhout/project/edge_infer/models/InternVLA-cosmos-encoder-batch6.pt2",
+        "device_name": "AMD Radeon(TM) 890M Graphics",
+        "placement_granularity": "output_device",
+        "session_providers": [
+          "torch-directml"
+        ],
+        "available_providers": [
+          "torch-directml"
+        ],
+        "node_counts": {
+          "dml": 1,
+          "cpu": 0
+        },
+        "total_nodes": 1,
+        "target_nodes": 1,
+        "fraction_on_target": 1.0,
+        "session_create_s": 1.4067443610401824,
+        "warmup_s": 0.42408207297557965,
+        "rss_bytes": 667328512,
+        "inputs": [
+          {
+            "name": "pixels",
+            "shape": [
+              6,
+              3,
+              256,
+              256
+            ]
+          }
+        ],
+        "outputs": [
+          {
+            "name": "out_0",
+            "shape": [
+              6,
+              16,
+              32,
+              32
+            ]
+          }
+        ],
+        "profile_path": "",
+        "ep_directory": "",
+        "requested_provider_missing": "",
+        "fallback_note": "",
+        "onnxruntime": "",
+        "note": ""
+      }
+    },
+    "loaded_rss_bytes": 9936982016,
+    "loaded_process_tree": [
+      {
+        "pid": 1247789,
+        "rss_bytes": 9269653504,
+        "private_bytes": 9269653504
+      },
+      {
+        "pid": 1247854,
+        "rss_bytes": 667328512,
+        "private_bytes": 667328512
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [],
+  "measured": [
+    {
+      "wall_s": 3.888449339021463,
+      "action_sha256": "fdeab4bace64e2b58811d9681f10249e26deb998ac55e3144634af68d6bb6b8d",
+      "stage_event": {
+        "request_id": "internvla-measured-0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "352c114f2b494f9384b2d561519db724",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "352c114f2b494f9384b2d561519db724",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      },
+      "action_metadata": {
+        "observation_timestamp_ns": 1790193231392094881,
+        "generation_timestamp_ns": 1790193235276499289,
+        "observation_age_s": 3.884404408,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "metrics": {
+        "policy_wall_s": 3.8843303869944066,
+        "worker_wall_s": 3.8761936689843424,
+        "cuda_peak_reserved_bytes": 0
+      }
+    }
+  ],
+  "nearest_rank_p50_wall_s": 3.888449339021463,
+  "nearest_rank_p95_wall_s": 3.888449339021463,
+  "unique_action_hashes": [
+    "fdeab4bace64e2b58811d9681f10249e26deb998ac55e3144634af68d6bb6b8d"
+  ],
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "352c114f2b494f9384b2d561519db724",
+    "worker_pid": 1247789,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+      "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+    },
+    "placement": "radeon-cosmos",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "radeon-cosmos",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "float32",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.232657331973314,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": {
+        "ep": "dml",
+        "graph_path": "/home/zhout/project/edge_infer/models/InternVLA-cosmos-encoder-batch6.pt2",
+        "device_name": "AMD Radeon(TM) 890M Graphics",
+        "placement_granularity": "output_device",
+        "session_providers": [
+          "torch-directml"
+        ],
+        "available_providers": [
+          "torch-directml"
+        ],
+        "node_counts": {
+          "dml": 1,
+          "cpu": 0
+        },
+        "total_nodes": 1,
+        "target_nodes": 1,
+        "fraction_on_target": 1.0,
+        "session_create_s": 1.4067443610401824,
+        "warmup_s": 0.42408207297557965,
+        "rss_bytes": 667328512,
+        "inputs": [
+          {
+            "name": "pixels",
+            "shape": [
+              6,
+              3,
+              256,
+              256
+            ]
+          }
+        ],
+        "outputs": [
+          {
+            "name": "out_0",
+            "shape": [
+              6,
+              16,
+              32,
+              32
+            ]
+          }
+        ],
+        "profile_path": "",
+        "ep_directory": "",
+        "requested_provider_missing": "",
+        "fallback_note": "",
+        "onnxruntime": "",
+        "note": ""
+      }
+    },
+    "loaded_rss_bytes": 9936982016,
+    "loaded_process_tree": [
+      {
+        "pid": 1247789,
+        "rss_bytes": 9269653504,
+        "private_bytes": 9269653504
+      },
+      {
+        "pid": 1247854,
+        "rss_bytes": 667328512,
+        "private_bytes": 667328512
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "reserved": {
+      "host_ram": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,322 @@
+/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
+ --> vLLM version 0.28.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+W0924 03:52:59.044000 1247715 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+W0924 03:52:59.080000 1247715 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+INFO 09-24 03:53:00 [importing.py:53] Triton is installed but 0 active driver(s) found (expected 1). Disabling Triton to prevent runtime errors.
+WARNING 09-24 03:53:00 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
+INFO 09-24 03:53:00 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
+INFO 09-24 03:53:02 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:53:02 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:53:02 [patch.py:646] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
+INFO 09-24 03:53:02 [patch.py:756] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
+INFO 09-24 03:53:06 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:53:06 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "radeon-cosmos",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+    "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "demands": {
+      "host_ram": 17179869184
+    }
+  },
+  "host_available_bytes_before": 31499673600,
+  "cuda_before": null,
+  "warmups": 0,
+  "repeats": 1,
+  "startup_s": 44.943421232979745,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "352c114f2b494f9384b2d561519db724",
+    "worker_pid": 1247789,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+      "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+    },
+    "placement": "radeon-cosmos",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "radeon-cosmos",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "float32",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.232657331973314,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": {
+        "ep": "dml",
+        "graph_path": "/home/zhout/project/edge_infer/models/InternVLA-cosmos-encoder-batch6.pt2",
+        "device_name": "AMD Radeon(TM) 890M Graphics",
+        "placement_granularity": "output_device",
+        "session_providers": [
+          "torch-directml"
+        ],
+        "available_providers": [
+          "torch-directml"
+        ],
+        "node_counts": {
+          "dml": 1,
+          "cpu": 0
+        },
+        "total_nodes": 1,
+        "target_nodes": 1,
+        "fraction_on_target": 1.0,
+        "session_create_s": 1.4067443610401824,
+        "warmup_s": 0.42408207297557965,
+        "rss_bytes": 667328512,
+        "inputs": [
+          {
+            "name": "pixels",
+            "shape": [
+              6,
+              3,
+              256,
+              256
+            ]
+          }
+        ],
+        "outputs": [
+          {
+            "name": "out_0",
+            "shape": [
+              6,
+              16,
+              32,
+              32
+            ]
+          }
+        ],
+        "profile_path": "",
+        "ep_directory": "",
+        "requested_provider_missing": "",
+        "fallback_note": "",
+        "onnxruntime": "",
+        "note": ""
+      }
+    },
+    "loaded_rss_bytes": 9936982016,
+    "loaded_process_tree": [
+      {
+        "pid": 1247789,
+        "rss_bytes": 9269653504,
+        "private_bytes": 9269653504
+      },
+      {
+        "pid": 1247854,
+        "rss_bytes": 667328512,
+        "private_bytes": 667328512
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [],
+  "nearest_rank_p50_wall_s": 3.888449339021463,
+  "nearest_rank_p95_wall_s": 3.888449339021463,
+  "unique_action_hashes": [
+    "fdeab4bace64e2b58811d9681f10249e26deb998ac55e3144634af68d6bb6b8d"
+  ],
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "352c114f2b494f9384b2d561519db724",
+    "worker_pid": 1247789,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516",
+      "graph": "9111b3841893acaaa11cf45715d246da3ac9cdd03c7b33fb3c7c20fe38d74704"
+    },
+    "placement": "radeon-cosmos",
+    "worker_props": {
+      "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8",
+      "placement": "radeon-cosmos",
+      "torch": "2.13.0+cpu",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "float32",
+      "policy_device": "cpu",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": null,
+      "cuda_device_index": null,
+      "cuda_allocated_after_load_bytes": null,
+      "cuda_reserved_after_load_bytes": null,
+      "policy_load_s": 29.232657331973314,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": {
+        "ep": "dml",
+        "graph_path": "/home/zhout/project/edge_infer/models/InternVLA-cosmos-encoder-batch6.pt2",
+        "device_name": "AMD Radeon(TM) 890M Graphics",
+        "placement_granularity": "output_device",
+        "session_providers": [
+          "torch-directml"
+        ],
+        "available_providers": [
+          "torch-directml"
+        ],
+        "node_counts": {
+          "dml": 1,
+          "cpu": 0
+        },
+        "total_nodes": 1,
+        "target_nodes": 1,
+        "fraction_on_target": 1.0,
+        "session_create_s": 1.4067443610401824,
+        "warmup_s": 0.42408207297557965,
+        "rss_bytes": 667328512,
+        "inputs": [
+          {
+            "name": "pixels",
+            "shape": [
+              6,
+              3,
+              256,
+              256
+            ]
+          }
+        ],
+        "outputs": [
+          {
+            "name": "out_0",
+            "shape": [
+              6,
+              16,
+              32,
+              32
+            ]
+          }
+        ],
+        "profile_path": "",
+        "ep_directory": "",
+        "requested_provider_missing": "",
+        "fallback_note": "",
+        "onnxruntime": "",
+        "note": ""
+      }
+    },
+    "loaded_rss_bytes": 9936982016,
+    "loaded_process_tree": [
+      {
+        "pid": 1247789,
+        "rss_bytes": 9269653504,
+        "private_bytes": 9269653504
+      },
+      {
+        "pid": 1247854,
+        "rss_bytes": 667328512,
+        "private_bytes": 667328512
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 0,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072
+    },
+    "reserved": {
+      "host_ram": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,18 @@
+/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
+ --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
+ --> vLLM version 0.28.0
+This will likely cause compatibility issues.
+  warn_if_misaligned_vllm_version()
+W0924 03:53:14.212000 1247789 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+W0924 03:53:14.235000 1247789 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
+WARNING 09-24 03:53:15 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
+INFO 09-24 03:53:15 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
+INFO 09-24 03:53:16 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:53:16 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:53:16 [patch.py:646] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
+INFO 09-24 03:53:16 [patch.py:756] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
+INFO 09-24 03:53:16 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=43575 props={"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "/home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "/home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8", "placement": "radeon-cosmos", "torch": "2.13.0+cpu", "policy_dtype": "bfloat16", "cosmos_dtype": "float32", "policy_device": "cpu", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": null, "cuda_device_index": null, "cuda_allocated_after_load_bytes": null, "cuda_reserved_after_load_bytes": null, "policy_load_s": 29.232657331973314, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": {"ep": "dml", "graph_path": "/home/zhout/project/edge_infer/models/InternVLA-cosmos-encoder-batch6.pt2", "device_name": "AMD Radeon(TM) 890M Graphics", "placement_granularity": "output_device", "session_providers": ["torch-directml"], "available_providers": ["torch-directml"], "node_counts": {"dml": 1, "cpu": 0}, "total_nodes": 1, "target_nodes": 1, "fraction_on_target": 1.0, "session_create_s": 1.4067443610401824, "warmup_s": 0.42408207297557965, "rss_bytes": 667328512, "inputs": [{"name": "pixels", "shape": [6, 3, 256, 256]}], "outputs": [{"name": "out_0", "shape": [6, 16, 32, 32]}], "profile_path": "", "ep_directory": "", "requested_provider_missing": "", "fallback_note": "", "onnxruntime": "", "note": ""}}
+127.0.0.1 - - [24/Sep/2026 03:53:51] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:53:51] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:53:55] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,47 @@
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 8589934592
+    }
+  },
+  "host_available_bytes_before": 31466561536,
+  "cuda_before": {
+    "free_bytes": 24242028544,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 0,
+  "repeats": 1,
+  "admission_refusal": "ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation",
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,71 @@
+INFO 09-24 03:27:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:27:34 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:27:34 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
+INFO 09-24 03:27:39 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:27:39 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
+ERROR 09-24 03:27:42 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
+ERROR 09-24 03:27:42 [stage_runtime.py:265] Traceback (most recent call last):
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
+ERROR 09-24 03:27:42 [stage_runtime.py:265]                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     raise primary_exc
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     client = self._initialize_replica(
+ERROR 09-24 03:27:42 [stage_runtime.py:265]              ^^^^^^^^^^^^^^^^^^^^^^^^^
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 817, in _initialize_replica
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     return create_graph_client(
+ERROR 09-24 03:27:42 [stage_runtime.py:265]            ^^^^^^^^^^^^^^^^^^^^
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/__init__.py", line 31, in create_graph_client
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     return InternVLAStageClient(metadata, backend, ledger, reservation)
+ERROR 09-24 03:27:42 [stage_runtime.py:265]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ERROR 09-24 03:27:42 [stage_runtime.py:265]   File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/internvla.py", line 120, in __init__
+ERROR 09-24 03:27:42 [stage_runtime.py:265]     raise ResourceUnavailable("InternVLA CUDA weights and workspace/headroom exceed reservation")
+ERROR 09-24 03:27:42 [stage_runtime.py:265] vllm_omni.engine.resource_ledger.ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "021044895e95be79dc2f110367607e684119afbc8ce75f6f0eec94844e0acec7",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 8589934592
+    }
+  },
+  "host_available_bytes_before": 31466561536,
+  "cuda_before": {
+    "free_bytes": 24242028544,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 0,
+  "repeats": 1,
+  "admission_refusal": "ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation",
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 30064771072,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,36 @@
+# Native Windows RTX 5090 Laptop: admitted InternVLA Omni action stage
+
+**Scoped synthetic policy path passed; robot-task E2E remains unqualified.** The real `InternVLA-A1-3B-FT-Place_Markpen` checkpoint (revision `0a557c6a503e6545b369bff2f34c76a410bbd190`) strict-loaded from the WSL UNC model path into the existing Omni `InternVLAA1Pipeline`. A bounded whole-policy graph stage kept the BF16 policy and Cosmos encoder on the NVIDIA GeForce RTX 5090 **Laptop** GPU under native Windows 11 build 26200. The output was one terminal float32 `[1,50,32]` host action chunk per request. Worker properties and runtime checks verified CUDA placement, exact model/interpreter artifact hashes, and absence of the external DirectML route.
+
+The host was Ryzen AI 9 HX 370 with NVIDIA driver 610.71. The worker used native Python 3.12.10, PyTorch 2.13.0+cu130, installed vLLM 0.29.0 and this Omni checkout. The fixture was three normalized patterned synthetic two-frame camera histories, valid masks, zero `[1,32]` state, the fixed Place_Markpen task and zero `[1,50,32]` diffusion noise. Neither this fixture nor the resulting actions establish physical task quality.
+
+After one warmup, **20/20 serial complete policy requests** produced the same finite action hash. Nearest-rank complete-request wall p50/p95 was **0.746/0.806 s** (range 0.698–0.806 s), excluding **147.77 s** controller/worker startup. Each measured request used the same resident worker. A separate [public `AsyncOmni.generate` request](public.json) passed with the same action hash and one terminal event; its first-request wall was 1.862 s after 155.16 s startup. These timing samples are not a comparison with the earlier direct zero-observation CUDA forward or the separately measured WSL CUDA profile.
+
+The declared budget was **16 GiB host RAM plus 16 GiB `cuda:0` VRAM** under 18/21 GiB ceilings. At the start of the [20-request profile](profile20.json), Windows had 21.58 GB available host RAM and CUDA reported 24.28 GB free. The worker reported 7.80 GB PyTorch CUDA reserved after load and a maximum **7.86 GB** peak reserved across measured requests. Sampled loaded process-tree RSS was 4.05 GB; it is not the host loading peak, and PyTorch's CUDA reservation is not a full-device VRAM trace. An [8 GiB VRAM demand](refusal.json) was rejected before worker launch because pinned weights plus declared workspace/headroom exceeded the reservation. In-flight cancellation produced no stale action and released both ledgers; shutdown left zero reservation or quarantine.
+
+The first measured action chunk differed from the separate native Windows CPU Omni action array by maximum absolute **0.07049** and relative L2 **2.340%**; see the exact values in the [profile report](profile20.json). Against the separately profiled [WSL CUDA action array](wsl_cuda_parity.json), relative L2 was **1.555%** and cosine 0.999886. No task tolerance has been established, so neither comparison proves task equivalence. Metadata explicitly says `action_mode=delta`, unverified physical units and joint order, unknown step time, and `control_ready=false`.
+
+The [driver](profile20_driver.log), [worker](profile20_worker.log), first measured [actions](profile20_actions.npy), [public logs](public_driver.log) and [refusal log](refusal_driver.log) are retained. The [preflight](preflight.json) documents available host/GPU memory before the Windows attempt; availability was sampled again by each probe. Real robot observations/reference actions, deadlines, post-cancel restart, concurrent admission, loading peak, sustained tail latency, power and thermals remain open.
+
+Reproduce in PowerShell with the pinned local artifacts:
+
+```powershell
+$root = '\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer'
+$repo = Join-Path $root 'vllm-omni-edge'
+$evidence = Join-Path $repo 'benchmarks\edge_harness\results\e2e_expansion_20260923\evidence\internvla_omni_windows_cuda'
+$py = 'C:\Users\zhout\w2\omni029venv\Scripts\python.exe'
+$env:PYTHONUTF8 = '1'
+$env:PYTHONPATH = $repo
+$env:VLLM_ENABLE_V1_MULTIPROCESSING = '0'
+& $py (Join-Path $repo 'benchmarks\edge_harness\experiments\probe_omni_internvla_policy.py') `
+  --model-dir (Join-Path $root 'models\InternVLA-A1-3B-FT-Place_Markpen') `
+  --processor-dir (Join-Path $root 'models\Qwen3-VL-2B-Instruct-processor') `
+  --cosmos-dir (Join-Path $root 'models\Cosmos-Tokenizer-CI8x8') `
+  --python-bin $py --placement cuda `
+  --capacity-gib 18 --reserve-gib 16 --vram-capacity-gib 21 --vram-reserve-gib 16 `
+  --warmups 1 --repeats 20 --abort-check `
+  --log-file (Join-Path $evidence 'profile20_worker.log') `
+  --output-report (Join-Path $evidence 'profile20.json') `
+  --output-actions (Join-Path $evidence 'profile20_actions.npy') `
+  --reference-actions (Join-Path $repo 'benchmarks\edge_harness\results\e2e_expansion_20260923\evidence\internvla_omni_windows_cpu\cpu_profile20_actions.npy')
+```
@@ -0,0 +1,10 @@
+{
+  "timestamp": "2026-09-24T03:36:31.154440+08:00",
+  "os": "Windows-11-10.0.26200-SP0",
+  "host_available_bytes": 20566712320,
+  "cuda_free_bytes": 24275582976,
+  "cuda_total_bytes": 25650855936,
+  "cuda_device": "NVIDIA GeForce RTX 5090 Laptop GPU",
+  "required_host_reservation_bytes": 17179869184,
+  "admissible": true
+}
@@ -0,0 +1,302 @@
+INFO 09-24 03:40:55 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:40:56 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:41:39 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:41:39 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout'
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "host_available_bytes_before": 21575876608,
+  "cuda_before": {
+    "free_bytes": 24275582976,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 1,
+  "repeats": 20,
+  "startup_s": 147.77138159994502,
+  "execution_plan_start": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "836d210216414303b28c873b9f9b163e",
+    "worker_pid": 48388,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cuda",
+    "worker_props": {
+      "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Cosmos-Tokenizer-CI8x8",
+      "placement": "cuda",
+      "torch": "2.13.0+cu130",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cuda",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU",
+      "cuda_device_index": 0,
+      "cuda_allocated_after_load_bytes": 6451405312,
+      "cuda_reserved_after_load_bytes": 7799308288,
+      "policy_load_s": 88.05892060010228,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 4045799424,
+    "loaded_process_tree": [
+      {
+        "pid": 48388,
+        "rss_bytes": 4382720,
+        "private_bytes": 778240
+      },
+      {
+        "pid": 57020,
+        "rss_bytes": 7462912,
+        "private_bytes": 1241088
+      },
+      {
+        "pid": 42520,
+        "rss_bytes": 4033953792,
+        "private_bytes": 13105008640
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 15373460732,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "warmup_results": [
+    {
+      "wall_s": 1.7163411000510678,
+      "action_sha256": "c03531d6c5b230e2bedb2f10af5183646f42462dc19bff13c34defce305977d2",
+      "stage_event": {
+        "request_id": "internvla-warmup-0",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "836d210216414303b28c873b9f9b163e",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "836d210216414303b28c873b9f9b163e",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      },
+      "action_metadata": {
+        "observation_timestamp_ns": 1790192646471288300,
+        "generation_timestamp_ns": 1790192648181790200,
+        "observation_age_s": 1.7105019,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "metrics": {
+        "policy_wall_s": 1.7087780999718234,
+        "worker_wall_s": 1.668760500033386,
+        "cuda_peak_reserved_bytes": 7862222848
+      }
+    }
+  ],
+  "nearest_rank_p50_wall_s": 0.7457928999792784,
+  "nearest_rank_p95_wall_s": 0.8055737001122907,
+  "unique_action_hashes": [
+    "c03531d6c5b230e2bedb2f10af5183646f42462dc19bff13c34defce305977d2"
+  ],
+  "output_actions": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260923\\evidence\\internvla_omni_windows_cuda\\profile20_actions.npy",
+  "reference_comparison": {
+    "max_abs": 0.07049129903316498,
+    "relative_l2": 0.02339840866625309,
+    "cosine": 0.9997268319129944
+  },
+  "execution_plan_end": {
+    "backend": "external.internvla.policy.v1",
+    "stage_id": 0,
+    "worker_generation": "836d210216414303b28c873b9f9b163e",
+    "worker_pid": 48388,
+    "worker_script_sha256": "7de45886e41d097466d47b25a715d7ec4acc688aab2c30fdc917c6ddea586632",
+    "artifact_sha256": {
+      "python": "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14",
+      "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+      "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+      "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+      "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+      "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+      "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+      "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+      "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+    },
+    "placement": "cuda",
+    "worker_props": {
+      "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
+      "processor_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3-VL-2B-Instruct-processor",
+      "cosmos_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Cosmos-Tokenizer-CI8x8",
+      "placement": "cuda",
+      "torch": "2.13.0+cu130",
+      "policy_dtype": "bfloat16",
+      "cosmos_dtype": "bfloat16",
+      "policy_device": "cuda",
+      "runtime_mode": "real_checkpoint_loaded",
+      "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU",
+      "cuda_device_index": 0,
+      "cuda_allocated_after_load_bytes": 6451405312,
+      "cuda_reserved_after_load_bytes": 7799308288,
+      "policy_load_s": 88.05892060010228,
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "image_shape": [
+        1,
+        2,
+        3,
+        224,
+        224
+      ],
+      "state_shape": [
+        1,
+        32
+      ],
+      "action_mode": "delta",
+      "action_units": "unverified",
+      "action_joint_order": "unverified",
+      "action_step_s": null,
+      "control_ready": false,
+      "external_load": null
+    },
+    "loaded_rss_bytes": 4045799424,
+    "loaded_process_tree": [
+      {
+        "pid": 48388,
+        "rss_bytes": 4382720,
+        "private_bytes": 778240
+      },
+      {
+        "pid": 57020,
+        "rss_bytes": 7462912,
+        "private_bytes": 1241088
+      },
+      {
+        "pid": 42520,
+        "rss_bytes": 4033953792,
+        "private_bytes": 13105008640
+      }
+    ],
+    "reserved_bytes": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    },
+    "memory_overhead_bytes": 8589934592,
+    "cuda_weights_and_headroom_min_bytes": 15373460732,
+    "max_input_bytes": 8388608,
+    "max_action_bytes": 1048576,
+    "request_capacity": 1,
+    "stateful_session": "InternVLA policy worker-owned; complete action chunk per request",
+    "evidence": "B"
+  },
+  "abort_check": {
+    "stale_output": false,
+    "ledger": {
+      "capacities": {
+        "host_ram": 19327352832,
+        "cuda:0": 22548578304
+      },
+      "reserved": {
+        "host_ram": 0,
+        "cuda:0": 0
+      },
+      "owners": [],
+      "quarantined": []
+    }
+  },
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,27 @@
+INFO 09-24 03:42:33 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:42:33 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:42:37 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=52402 props={"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Cosmos-Tokenizer-CI8x8", "placement": "cuda", "torch": "2.13.0+cu130", "policy_dtype": "bfloat16", "cosmos_dtype": "bfloat16", "policy_device": "cuda", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU", "cuda_device_index": 0, "cuda_allocated_after_load_bytes": 6451405312, "cuda_reserved_after_load_bytes": 7799308288, "policy_load_s": 88.05892060010228, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": null}
+127.0.0.1 - - [24/Sep/2026 03:44:06] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:06] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:08] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:08] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:09] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:10] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:11] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:11] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:12] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:13] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:14] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:15] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:15] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:16] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:17] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:18] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:18] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:19] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:20] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:21] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:21] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:22] "POST /v1/actions HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:44:23] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,78 @@
+{
+  "entrypoint": "AsyncOmni.generate",
+  "placement": "cuda",
+  "pipeline": "internvla_a1_external_whole",
+  "host_available_bytes_before": 24073027584,
+  "cuda_before": {
+    "free_bytes": 24275582976,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "startup_s": 155.15575060003903,
+  "request_wall_s": 1.8624655000166968,
+  "outputs": [
+    {
+      "request_id": "public-internvla-1",
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "action_sha256": "c03531d6c5b230e2bedb2f10af5183646f42462dc19bff13c34defce305977d2",
+      "finite": true,
+      "metadata": {
+        "observation_timestamp_ns": 1790192998287945000,
+        "generation_timestamp_ns": 1790193000130250000,
+        "observation_age_s": 1.842305,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "stage_event": {
+        "request_id": "public-internvla-1-bb5b801c",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "9058c78591ec49f29986103d7d35c05b",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "9058c78591ec49f29986103d7d35c05b",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      }
+    }
+  ],
+  "status": "passed"
+}
@@ -0,0 +1,96 @@
+INFO 09-24 03:46:43 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:46:43 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:47:23 [omni_base.py:216] [AsyncOmni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
+INFO 09-24 03:47:23 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
+ERROR 09-24 03:47:23 [repo_utils.py:119] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen: Unrecognized model in \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen. Should have a `model_type` key in its config.json., retrying 1 of 2
+ERROR 09-24 03:47:25 [repo_utils.py:117] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen: Unrecognized model in \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen. Should have a `model_type` key in its config.json.
+INFO 09-24 03:47:25 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
+INFO 09-24 03:47:25 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+INFO 09-24 03:47:25 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
+INFO 09-24 03:49:58 [orchestrator.py:604] [Orchestrator] Starting event loop
+INFO 09-24 03:49:58 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
+INFO 09-24 03:49:58 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 155.15 seconds
+INFO 09-24 03:49:58 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
+INFO 09-24 03:50:00 [omni_base.py:781] [AsyncOmni] Shutting down
+INFO 09-24 03:50:00 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
+INFO 09-24 03:50:00 [orchestrator.py:705] [Orchestrator] Received shutdown signal
+INFO 09-24 03:50:00 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
+INFO 09-24 03:50:00 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
+{
+  "entrypoint": "AsyncOmni.generate",
+  "placement": "cuda",
+  "pipeline": "internvla_a1_external_whole",
+  "host_available_bytes_before": 24073027584,
+  "cuda_before": {
+    "free_bytes": 24275582976,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 17179869184
+    }
+  },
+  "startup_s": 155.15575060003903,
+  "request_wall_s": 1.8624655000166968,
+  "outputs": [
+    {
+      "request_id": "public-internvla-1",
+      "action_shape": [
+        1,
+        50,
+        32
+      ],
+      "action_sha256": "c03531d6c5b230e2bedb2f10af5183646f42462dc19bff13c34defce305977d2",
+      "finite": true,
+      "metadata": {
+        "observation_timestamp_ns": 1790192998287945000,
+        "generation_timestamp_ns": 1790193000130250000,
+        "observation_age_s": 1.842305,
+        "action_mode": "delta",
+        "action_shape": [
+          1,
+          50,
+          32
+        ],
+        "action_units": "unverified",
+        "joint_order": "unverified",
+        "action_step_s": null,
+        "control_ready": false
+      },
+      "stage_event": {
+        "request_id": "public-internvla-1-bb5b801c",
+        "stage_id": 0,
+        "epoch": 1,
+        "seq": 1,
+        "kind": "action",
+        "worker_generation": "9058c78591ec49f29986103d7d35c05b",
+        "buffers": [
+          {
+            "object_id": "actions",
+            "owner": "0",
+            "generation": "9058c78591ec49f29986103d7d35c05b",
+            "dtype": "float32",
+            "shape": [
+              1,
+              50,
+              32
+            ],
+            "nbytes": 6400,
+            "memory_domain": "host-copy",
+            "layout": "C",
+            "ready": true
+          }
+        ],
+        "terminal": true,
+        "error": null
+      }
+    }
+  ],
+  "status": "passed"
+}
@@ -0,0 +1,7 @@
+INFO 09-24 03:48:19 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:48:19 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:48:24 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
+internvla-worker ready port=49727 props={"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", "processor_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3-VL-2B-Instruct-processor", "cosmos_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Cosmos-Tokenizer-CI8x8", "placement": "cuda", "torch": "2.13.0+cu130", "policy_dtype": "bfloat16", "cosmos_dtype": "bfloat16", "policy_device": "cuda", "runtime_mode": "real_checkpoint_loaded", "cuda_device_name": "NVIDIA GeForce RTX 5090 Laptop GPU", "cuda_device_index": 0, "cuda_allocated_after_load_bytes": 6451405312, "cuda_reserved_after_load_bytes": 7799308288, "policy_load_s": 93.18841519998387, "action_shape": [1, 50, 32], "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32], "action_mode": "delta", "action_units": "unverified", "action_joint_order": "unverified", "action_step_s": null, "control_ready": false, "external_load": null}
+127.0.0.1 - - [24/Sep/2026 03:49:58] "GET /health HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:49:58] "GET /props HTTP/1.1" 200 -
+127.0.0.1 - - [24/Sep/2026 03:50:00] "POST /v1/actions HTTP/1.1" 200 -
@@ -0,0 +1,47 @@
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 8589934592
+    }
+  },
+  "host_available_bytes_before": 25076883456,
+  "cuda_before": {
+    "free_bytes": 24275582976,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 0,
+  "repeats": 1,
+  "admission_refusal": "ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation",
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,88 @@
+INFO 09-24 03:44:58 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
+INFO 09-24 03:44:58 [patch.py:510] inductor factorable-divisibility patch: installed.
+INFO 09-24 03:45:38 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
+WARNING 09-24 03:45:38 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout'
+ERROR 09-24 03:46:15 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265] Traceback (most recent call last):
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     raise primary_exc
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     client = self._initialize_replica(
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]              ^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     return create_graph_client(
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]            ^^^^^^^^^^^^^^^^^^^^
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 31, in create_graph_client
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     return InternVLAStageClient(metadata, backend, ledger, reservation)
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]   File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\internvla.py", line 120, in __init__
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265]     raise ResourceUnavailable("InternVLA CUDA weights and workspace/headroom exceed reservation")
+
+ERROR 09-24 03:46:15 [stage_runtime.py:265] vllm_omni.engine.resource_ledger.ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation
+{
+  "scope": "real InternVLA Place_Markpen checkpoint via bounded Omni whole-policy graph stage; synthetic patterned observations/noise; no robot-task quality claim",
+  "placement": "cuda",
+  "artifact_sha256": {
+    "python": "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14",
+    "model": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
+    "model_config": "1272ca8fb93cb42f48b039bb298688b2bd50a66f4faa0afcd9e538e9762f48aa",
+    "train_config": "e50ec42e40ba1925983e0e8db36558d3738bea64e1698045ff7cc08a130e1053",
+    "stats": "354710650bb91fc0e84a461987b3554b30bef9d5de47567480624484cdfbf3a2",
+    "cosmos_encoder": "520a3b878e72ee872faccb1211b5d622d194c0fdd177f2f6df097586b8fa98e5",
+    "cosmos_decoder": "3b7791e6c8f6f92caacddf48ebb314f044bcb8302d0c7f47501d384a7a80f738",
+    "processor_tokenizer": "a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7",
+    "processor_config": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+  },
+  "budget": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "demands": {
+      "host_ram": 17179869184,
+      "cuda:0": 8589934592
+    }
+  },
+  "host_available_bytes_before": 25076883456,
+  "cuda_before": {
+    "free_bytes": 24275582976,
+    "total_bytes": 25650855936,
+    "device_name": "NVIDIA GeForce RTX 5090 Laptop GPU"
+  },
+  "warmups": 0,
+  "repeats": 1,
+  "admission_refusal": "ResourceUnavailable: InternVLA CUDA weights and workspace/headroom exceed reservation",
+  "status": "passed",
+  "ledger_after_shutdown": {
+    "capacities": {
+      "host_ram": 19327352832,
+      "cuda:0": 22548578304
+    },
+    "reserved": {
+      "host_ram": 0,
+      "cuda:0": 0
+    },
+    "owners": [],
+    "quarantined": []
+  }
+}
@@ -0,0 +1,15 @@
+{
+  "scope": "same patterned synthetic Place_Markpen observation/noise; no robot task tolerance",
+  "reference": "WSL CUDA Omni first measured actions",
+  "candidate": "native Windows CUDA Omni first measured actions",
+  "reference_file_sha256": "c4157bd68b586611f96919a571e9f91fc3c2f6d06921d0e13383ed4d65201e24",
+  "candidate_file_sha256": "586653ea5638651842ffb30678ee8735a05d29ed2dab569c5204fd2e60772ad0",
+  "shape": [
+    1,
+    50,
+    32
+  ],
+  "max_abs": 0.013449281454086304,
+  "relative_l2": 0.015554326586425304,
+  "cosine": 0.9998864531517029
+}
@@ -2,7 +2,7 @@

 All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, native Windows CPU Qwen3-TTS text-to-WAV profiles through both standalone PyTorch and a bounded Omni whole-session stage, and a hybrid CPU+Radeon 890M Qwen3-TTS profile likewise repeated through a bounded Omni whole-session stage. A pinned GGUF MiniCPM-o route adds combined image+audio to text+speech runs on WSL CPU, native Windows CPU and native Windows Radeon 890M+CPU. All three now pass a bounded complete request through an Omni graph stage; the WSL CPU path additionally passed one warmup plus 20 serial measured requests. These are scoped synthetic cases, not full release qualification.

-The WSL CPU and CPU+Radeon InternVLA cells also have [bounded Omni complete-policy profiles](../../../e2e_expansion_20260923/evidence/internvla_omni_policy/README.md): one warmup and 20 serial patterned synthetic requests per placement, plus admission refusal, cancellation and a public hybrid `AsyncOmni.generate` request. Separate native Windows [CPU](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cpu/README.md) and [CPU+Radeon](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_hybrid/README.md) Omni profiles add 20 serial patterned requests per placement, refusal, cancellation and public action requests. These advance only the synthetic action-policy path. Real observations/reference actions, physical action units/order/step time, task quality, loading peak and sustained power/thermal behavior remain unverified. The earlier [Cosmos component and direct hybrid results](../../../e2e_expansion_20260923/evidence/internvla_cosmos_radeon890m/README.md) remain separate experiments. A new [20-request native Radeon MiniCPM-o profile](../../../e2e_expansion_20260923/evidence/minicpmo_omni_radeon_profile20/README.md) passed the scoped combined input/output path but exposed a large memory-sensitive latency regression versus its earlier one-request smoke; its row below supersedes the smoke for repeated-request timing.
+The WSL CPU and CPU+Radeon InternVLA cells also have [bounded Omni complete-policy profiles](../../../e2e_expansion_20260923/evidence/internvla_omni_policy/README.md): one warmup and 20 serial patterned synthetic requests per placement, plus admission refusal, cancellation and a public hybrid `AsyncOmni.generate` request. The [WSL RTX 5090 Laptop CUDA](../../../e2e_expansion_20260923/evidence/internvla_omni_cuda/README.md) and [native Windows RTX CUDA](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cuda/README.md) cells now have the same scoped Omni whole-policy pattern with 20 serial requests each, explicit host/VRAM admission, refusal, cancellation and public action requests. Separate native Windows [CPU](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cpu/README.md) and [CPU+Radeon](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_hybrid/README.md) Omni profiles add 20 serial patterned requests per placement, refusal, cancellation and public action requests. These advance only the synthetic action-policy path. Real observations/reference actions, physical action units/order/step time, task quality, loading peak and sustained power/thermal behavior remain unverified. The earlier [Cosmos component and direct hybrid results](../../../e2e_expansion_20260923/evidence/internvla_cosmos_radeon890m/README.md) remain separate experiments. A new [20-request native Radeon MiniCPM-o profile](../../../e2e_expansion_20260923/evidence/minicpmo_omni_radeon_profile20/README.md) passed the scoped combined input/output path but exposed a large memory-sensitive latency regression versus its earlier one-request smoke; its row below supersedes the smoke for repeated-request timing.

 ## Configuration × model

@@ -18,8 +18,8 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 |---|---|---|---|---|---|
 | PC HX370: CPU / WSL | profile completed | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU |
 | PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF image+audio → text+speech (unstable latency) | synthetic policy pass: Omni CPU |
-| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass |
-| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass |
+| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
+| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
 | PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | scoped E2E: Omni Radeon+CPU image+audio → text+speech (unstable latency) | synthetic policy pass: Omni CPU+Radeon Cosmos (WSL/native Windows) |
 | PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
 | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
@@ -47,12 +47,12 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
 | PC HX370 + RTX 5090 Laptop / WSL | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
 | PC HX370 + RTX 5090 Laptop / WSL | Qwen3.8-27B | SCOPED_E2E | A new one-stage Omni binding ran the real NVFP4 checkpoint: one 16-token text request and one synthetic red-image request (answer `Red`) at an explicit 512-token context with host offload. This is not the full multimodal, long-context or quality workload. | Qualify image/video suites, longer contexts, latency distributions and sustained power; [recovery evidence](../../../e2e_recovery_20260923/README.md). |
 | PC HX370 + RTX 5090 Laptop / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed three-stage text, one synthetic image and one synthetic audio input to text+speech under separate constrained plans with 5 GiB host offload. The image response correctly described a 448×448 red square on white and produced a nonzero 5.2 s WAV in 17.49 s; a 2 s/440 Hz tone was described as “A loud whistling sound” with a nonzero 2.16 s WAV in 7.78 s. The original 0.58 thinker GPU budget failed image/KV admission; 0.61 admitted the same context and image. Image shutdown force-killed stage 0, and both multimodal runs reported shared-memory cleanup warnings. General image/audio quality and speech meaning remain unverified. | Validate real image/audio suites, video input, speech intelligibility, interruption, repeated latency, clean shutdown and sustained power; [text evidence](../../../e2e_recovery_20260923/README.md), [image/audio evidence](../../../e2e_expansion_20260923/README.md). |
-| PC HX370 + RTX 5090 Laptop / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | Strict real-weight Place_Markpen load and one policy forward on zero observations/noise produced finite `[1, 50, 32]` actions in 1.222 s. Synthetic input and one timing sample do not establish robot-task E2E or action semantics. | Run real observation/reference-action cases and validate units, timestamps and deadline behavior; [raw result](../../../e2e_recovery_20260923/evidence/internvla/internvla_finetune_synthetic.json). |
+| PC HX370 + RTX 5090 Laptop / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. One warmup plus 20/20 serial patterned synthetic observation/state/noise requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.382/0.399 s, excluding 43.82 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 28/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. An in-flight abort and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate action request passed. Relative to a separate patterned WSL CPU output, action relative L2 difference was 2.006% with no task tolerance. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_cuda/README.md), [earlier direct result](../../../e2e_recovery_20260923/evidence/internvla/internvla_finetune_synthetic.json). |
 | PC HX370 + RTX 5090 Laptop / Windows | Spark-X2.5 | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Broaden only with a named workload/device/precision qualification. |
 | PC HX370 + RTX 5090 Laptop / Windows | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
 | PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The Omni pipeline binding now exists and passed scoped text/image cases in WSL, but no native Windows Omni run has verified the NVFP4 checkpoint, memory admission or modality behavior. | Run the same checkpoint and explicit context on native Windows, then broaden quality and profiling; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
 | PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed native Windows three-stage BF16 text, one synthetic image, and one synthetic audio input to text+speech in separate constrained plans with 5 GiB thinker host offload and explicit 2,048/512-token limits. Text and all 32 thinker token IDs matched WSL CUDA; a 5.2 s WAV was nonzero. The red-square image produced the same description and 27 thinker token IDs as WSL plus a nonzero 5.2 s WAV in 17.15 s. The 2 s/440 Hz tone produced the same whistling description and nine IDs as WSL plus a nonzero 2.16 s WAV in 8.26 s. The initial text run needed 900/600 s startup limits after a 300 s timeout; a separate 1-warmup/20-measured serial text run had request wall p50 18.38 s/p95 18.73 s, with sampled host available RAM as low as 0.78 GB and pagefile use up to 11.17 GB. These synthetic inputs do not establish general image/audio or speech quality. | Check real image/audio suites, video input, speech intelligibility/alignment, concurrency, interruption, loading-peak admission and sustained thermal/power behavior before broader qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
-| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CUDA placement was checked; one native forward took 1.12 s, with 6.68 GB peak allocated. Relative L2 difference from WSL CUDA was 0.83% (no task tolerance). | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
+| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted native Windows Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. After one warmup, 20/20 serial patterned synthetic requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.746/0.806 s, excluding 147.77 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 18/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. In-flight cancellation and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate request returned the same action hash. Relative to the separate native CPU patterned output, action relative L2 difference was 2.340% with no task tolerance; native-vs-WSL CUDA relative L2 was 1.555%. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni Windows CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cuda/README.md), [earlier direct result](../../../e2e_expansion_20260923/README.md). |
 | PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The same pinned Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode through a bounded Omni whole-session stage with llama.cpp Vulkan1. The log identifies Radeon 890M and 29/29 layers offloaded there, rather than RTX 5090; an intentionally wrong device name was refused. `Paris` and `1511` passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 2.10/2.14 s. A public AsyncOmni request passed; an oversized context was rejected with a later correct request; in-flight cancellation produced no stale output and released the ledger. The prior standalone run was 2.18/2.20 s. No streaming, restart-after-cancel, GPU-memory peak or sustained power/thermal check is claimed. | Qualify public configuration, broader text/token quality, streaming and cancellation recovery, shared-RAM/GPU peak and sustained use; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
 | PC HX370 + Radeon 890M / Windows worker | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST_HYBRID | The pinned Q8_0 CustomVoice talker, F16 codec and punctuation GGUFs completed two named text-to-WAV requests through an Omni whole-session CrispASR stage on native Windows. The runtime log identifies Radeon 890M Vulkan0 for talker/codec and the 432 MB code predictor pinned to CPU FP32; this is not all-Radeon or NPU execution. Both PCM hashes exactly match the earlier standalone outputs. Whisper tiny.en transcribed the first Omni WAV and both standalone WAVs exactly (WER 0 as an intelligibility proxy). One warmup plus 20 serial complete Omni requests for 2.56 s of audio had identical PCM and nearest-rank wall p50/p95 3.298/3.336 s (RTF 1.29/1.30). A public AsyncOmni audio request passed; cancellation drained the worker without stale output or remaining reservation. The earlier standalone profile was 3.310/3.345 s. The default Vulkan configuration previously fell back to CPU, all-Vulkan ran away to a 360-frame cap, and DirectML failed waveform quality. No playable stream, post-cancel restart, loading peak, broad speech quality or sustained power/thermal run is claimed. | Validate playable streaming, restart/recovery, broader voice quality, shared-RAM/GPU peak, concurrency and sustained real-time behavior before wider qualification; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_hybrid/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML route](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_directml/README.md). |
 | PC HX370 + Radeon 890M / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | FP32 448x448 vision tower: DirectML 85/113 nodes; normalized max error 9.095e-06 versus torch CPU. Historical component path, not full 27B or current graph-stage image-to-text integration. | Recover the hashed tower bundle and qualify its current Omni graph handoff with the language stage. |
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-"""Bounded whole-policy InternVLA stage on CPU or CPU plus Radeon Cosmos.
+"""Bounded whole-policy InternVLA stage on CPU, CUDA or CPU plus Radeon Cosmos.

 The existing Omni diffusion policy owns action semantics. StageRuntime owns
 shared-RAM admission, one in-flight request, terminal output and cancellation.
@@ -65,9 +65,10 @@ class InternVLAStageClient(CrispTTSStageClient):
        self._request_timeout_s = float(config.get("request_timeout_s", 60))
        self._port = int(config.get("port") or _local_port())
        self._placement = str(config.get("placement", ""))
+        self._cuda_demand_bytes = reservation.demands.get("cuda:0", 0)
        try:
            if (
-                self._placement not in {"cpu", "radeon-cosmos"} or self._max_input_bytes <= 0
+                self._placement not in {"cpu", "cuda", "radeon-cosmos"} or self._max_input_bytes <= 0
                or self._max_action_bytes <= 0 or self._request_timeout_s <= 0
                or not 0 < self._port < 65536
            ):
@@ -75,6 +76,8 @@ class InternVLAStageClient(CrispTTSStageClient):
            if self._memory_pool not in reservation.demands:
                raise ResourceUnavailable("InternVLA shared-RAM pool absent from stage reservation")
            demand = reservation.demands[self._memory_pool]
+            if self._placement == "cuda" and reservation.demands.get("cuda:0", 0) <= 0:
+                raise ResourceUnavailable("InternVLA CUDA placement requires an explicit cuda:0 reservation")
            if self._max_input_bytes + self._max_action_bytes > demand:
                raise ResourceUnavailable("InternVLA I/O bound exceeds stage reservation")

@@ -109,6 +112,12 @@ class InternVLAStageClient(CrispTTSStageClient):
            artifact_bytes = sum(path.stat().st_size for name, path in artifacts.items() if name != "python")
            if overhead <= 0 or artifact_bytes + overhead > demand:
                raise ResourceUnavailable("InternVLA weights, graph and state/workspace/headroom exceed reservation")
+            cuda_min_bytes = 0
+            if self._placement == "cuda":
+                cuda_min_bytes = (artifacts["model"].stat().st_size
+                                  + artifacts["cosmos_encoder"].stat().st_size + overhead)
+                if cuda_min_bytes > self._cuda_demand_bytes:
+                    raise ResourceUnavailable("InternVLA CUDA weights and workspace/headroom exceed reservation")

            self._log_path = Path(config["log_file"]).resolve()
            self._log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -117,7 +126,8 @@ class InternVLAStageClient(CrispTTSStageClient):
            repo_root = Path(__file__).resolve().parents[3]
            env.update({
                "PYTHONPATH": str(repo_root) + os.pathsep + env.get("PYTHONPATH", ""),
-                "CUDA_VISIBLE_DEVICES": "", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1",
+                "CUDA_VISIBLE_DEVICES": "0" if self._placement == "cuda" else "",
+                "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1",
                "OMP_NUM_THREADS": "8", "MKL_NUM_THREADS": "8", "PYTHONUTF8": "1",
                "TOKENIZERS_PARALLELISM": "false",
            })
@@ -160,7 +170,7 @@ class InternVLAStageClient(CrispTTSStageClient):
                or Path(props.get("model_dir", "")).resolve() != self._model_dir
                or Path(props.get("processor_dir", "")).resolve() != self._processor_dir
                or Path(props.get("cosmos_dir", "")).resolve() != self._cosmos_dir
-                or props.get("policy_device") != "cpu"
+                or props.get("policy_device") != ("cuda" if self._placement == "cuda" else "cpu")
                or props.get("runtime_mode") != "real_checkpoint_loaded"
                or props.get("policy_dtype") != "bfloat16"
                or props.get("cosmos_dtype") != ("float32" if expected_external else "bfloat16")
@@ -170,6 +180,11 @@ class InternVLAStageClient(CrispTTSStageClient):
                or props.get("torch") != str(config["expected_torch"])
                or bool(observed_external) != expected_external
                or (expected_external and observed_external.get("device_name") != "AMD Radeon(TM) 890M Graphics")
+                or (self._placement == "cuda" and (
+                    props.get("cuda_device_name") != config.get("expected_cuda_device_name")
+                    or props.get("cuda_device_index") != 0
+                    or not 0 < props.get("cuda_reserved_after_load_bytes", 0) <= reservation.demands["cuda:0"]
+                ))
            ):
                raise RuntimeError("InternVLA worker model, precision or device differs from plan")
            import psutil
@@ -199,6 +214,7 @@ class InternVLAStageClient(CrispTTSStageClient):
                "loaded_process_tree": self._loaded_process_tree,
                "reserved_bytes": dict(reservation.demands),
                "memory_overhead_bytes": overhead,
+                "cuda_weights_and_headroom_min_bytes": cuda_min_bytes,
                "max_input_bytes": self._max_input_bytes,
                "max_action_bytes": self._max_action_bytes,
                "request_capacity": 1,
@@ -268,17 +284,23 @@ class InternVLAStageClient(CrispTTSStageClient):
            )
            with np.load(io.BytesIO(data), allow_pickle=False) as response:
                if set(response.files) != {"actions", "observation_timestamp_ns",
-                                          "generation_timestamp_ns", "request_id", "worker_wall_s"}:
+                                          "generation_timestamp_ns", "request_id", "worker_wall_s",
+                                          "cuda_peak_reserved_bytes"}:
                    raise ValueError("InternVLA response fields differ from declared action contract")
                actions = np.asarray(response["actions"])
                observation_ns = int(response["observation_timestamp_ns"].item())
                generation_ns = int(response["generation_timestamp_ns"].item())
                returned_id = str(response["request_id"].item())
                worker_wall_s = float(response["worker_wall_s"].item())
+                cuda_peak_reserved = int(response["cuda_peak_reserved_bytes"].item())
            if (returned_id != request.request_id or actions.dtype != np.float32
                or actions.shape != (1, 50, 32) or not np.isfinite(actions).all()
                or generation_ns < observation_ns):
                raise ValueError("InternVLA worker returned invalid, stale or misrouted actions")
+            if self._placement == "cuda" and not 0 < cuda_peak_reserved <= self._cuda_demand_bytes:
+                raise ResourceUnavailable("InternVLA CUDA peak reservation exceeded the admitted VRAM budget")
+            if self._placement != "cuda" and cuda_peak_reserved != 0:
+                raise ValueError("InternVLA non-CUDA worker reported CUDA allocation")
            ref = BufferRef(
                "actions", str(self.stage_id), self._generation, "float32", tuple(actions.shape), int(actions.nbytes)
            )
@@ -301,7 +323,8 @@ class InternVLAStageClient(CrispTTSStageClient):
                    },
                },
                metrics={"policy_wall_s": time.perf_counter() - started,
-                         "worker_wall_s": worker_wall_s},
+                         "worker_wall_s": worker_wall_s,
+                         "cuda_peak_reserved_bytes": cuda_peak_reserved},
            )
        except asyncio.CancelledError:
            raise
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-"""CPU InternVLA action worker with an optional Radeon Cosmos encoder.
+"""InternVLA action worker on CPU/CUDA with an optional Radeon Cosmos encoder.

 The policy remains the existing Omni diffusion implementation. Requests carry
 explicit normalized camera histories and state; the worker owns policy state
@@ -33,7 +33,7 @@ def main() -> None:
    parser.add_argument("--model-dir", type=Path, required=True)
    parser.add_argument("--processor-dir", type=Path, required=True)
    parser.add_argument("--cosmos-dir", type=Path, required=True)
-    parser.add_argument("--placement", choices=("cpu", "radeon-cosmos"), required=True)
+    parser.add_argument("--placement", choices=("cpu", "cuda", "radeon-cosmos"), required=True)
    parser.add_argument("--graph", type=Path)
    parser.add_argument("--graph-sha256")
    parser.add_argument("--port", type=int, required=True)
@@ -44,8 +44,9 @@ def main() -> None:
        parser.error("invalid port, threads or input bound")
    if args.placement == "radeon-cosmos" and (args.graph is None or not args.graph_sha256):
        parser.error("Radeon route requires a graph and hash")
-    if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
-        raise RuntimeError("InternVLA CPU worker requires CUDA_VISIBLE_DEVICES empty")
+    expected_visible = "0" if args.placement == "cuda" else ""
+    if os.environ.get("CUDA_VISIBLE_DEVICES") != expected_visible:
+        raise RuntimeError("InternVLA CUDA visibility differs from the placement plan")
    if os.environ.get("HF_HUB_OFFLINE") != "1":
        raise RuntimeError("InternVLA worker requires offline checkpoint loading")

@@ -58,6 +59,9 @@ def main() -> None:
    import numpy as np
    import torch

+    if args.placement == "cuda" and not torch.cuda.is_available():
+        raise RuntimeError("InternVLA CUDA placement requested but CUDA is unavailable")
+
    from vllm_omni.diffusion.data import OmniDiffusionConfig
    from vllm_omni.diffusion.registry import initialize_model
    from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -65,10 +69,11 @@ def main() -> None:
    from vllm_omni.inputs.data import OmniDiffusionSamplingParams

    torch.set_num_threads(args.threads)
+    policy_device = "cuda" if args.placement == "cuda" else "cpu"
    config = OmniDiffusionConfig(
        model=str(model_dir), model_class_name="InternVLAA1Pipeline", dtype=torch.bfloat16,
        custom_pipeline_args={
-            "device": "cpu", "dtype": "bfloat16", "compile_model": False,
+            "device": policy_device, "dtype": "bfloat16", "compile_model": False,
            "enable_regional_compile": False, "enable_warmup": False,
            "strict_load": True, "processor_model_name": str(processor_dir),
        },
@@ -78,8 +83,8 @@ def main() -> None:
    policy_load_s = time.perf_counter() - started
    if pipeline.runtime_mode() != "real_checkpoint_loaded":
        raise RuntimeError("InternVLA real checkpoint did not load")
-    if {parameter.device.type for parameter in pipeline.policy.parameters()} != {"cpu"}:
-        raise RuntimeError("InternVLA policy parameters did not remain on CPU")
+    if {parameter.device.type for parameter in pipeline.policy.parameters()} != {policy_device}:
+        raise RuntimeError("InternVLA policy parameters differ from the placement plan")

    external = None
    external_report = None
@@ -127,7 +132,11 @@ def main() -> None:
        "cosmos_dir": str(cosmos_dir), "placement": args.placement,
        "torch": torch.__version__, "policy_dtype": "bfloat16",
        "cosmos_dtype": "float32" if external is not None else "bfloat16",
-        "policy_device": "cpu", "runtime_mode": pipeline.runtime_mode(),
+        "policy_device": policy_device, "runtime_mode": pipeline.runtime_mode(),
+        "cuda_device_name": torch.cuda.get_device_name(0) if policy_device == "cuda" else None,
+        "cuda_device_index": torch.cuda.current_device() if policy_device == "cuda" else None,
+        "cuda_allocated_after_load_bytes": torch.cuda.memory_allocated(0) if policy_device == "cuda" else None,
+        "cuda_reserved_after_load_bytes": torch.cuda.memory_reserved(0) if policy_device == "cuda" else None,
        "policy_load_s": policy_load_s, "action_shape": [1, 50, 32],
        "image_shape": [1, 2, 3, 224, 224], "state_shape": [1, 32],
        "action_mode": "delta", "action_units": "unverified",
@@ -195,17 +204,17 @@ def main() -> None:
                    raise ValueError("state must be finite float32 [1,32]")
                if noise.dtype != np.float32 or noise.shape != (1, 50, 32) or not np.isfinite(noise).all():
                    raise ValueError("noise must be finite float32 [1,50,32]")
-                batch = {"observation.state": torch.from_numpy(state.copy()).to(torch.bfloat16),
+                batch = {"observation.state": torch.from_numpy(state.copy()).to(device=policy_device, dtype=torch.bfloat16),
                         "observation.task": [task]}
                for i in range(3):
-                    batch[f"observation.images.image{i}"] = torch.from_numpy(images[i].copy()).to(torch.bfloat16)
-                    batch[f"observation.images.image{i}_mask"] = torch.from_numpy(masks[i].copy())
+                    batch[f"observation.images.image{i}"] = torch.from_numpy(images[i].copy()).to(device=policy_device, dtype=torch.bfloat16)
+                    batch[f"observation.images.image{i}_mask"] = torch.from_numpy(masks[i].copy()).to(policy_device)
                started = time.perf_counter()
                result = pipeline.forward(DiffusionRequestBatch(requests=[
                    OmniDiffusionRequest(
                        prompt="",
                        sampling_params=OmniDiffusionSamplingParams(extra_args={
-                            "batch_inputs": batch, "noise": torch.from_numpy(noise.copy()),
+                            "batch_inputs": batch, "noise": torch.from_numpy(noise.copy()).to(policy_device),
                            "decode_image": False,
                        }),
                        request_id=request_id,
@@ -214,15 +223,19 @@ def main() -> None:
                if result.error:
                    raise RuntimeError(result.error)
                actions = result.output["payload"]["actions"]
-                if actions.device.type != "cpu" or tuple(actions.shape) != (1, 50, 32):
+                if actions.device.type != policy_device or tuple(actions.shape) != (1, 50, 32):
                    raise RuntimeError("policy action device or shape differs from plan")
                values = actions.detach().float().cpu().contiguous().numpy()
+                if policy_device == "cuda":
+                    torch.cuda.synchronize(0)
+                cuda_peak_reserved = torch.cuda.max_memory_reserved(0) if policy_device == "cuda" else 0
                if not np.isfinite(values).all():
                    raise RuntimeError("policy returned nonfinite actions")
                with io.BytesIO() as out:
                    np.savez(out, actions=values, observation_timestamp_ns=np.int64(observation_ns),
                             generation_timestamp_ns=np.int64(time.time_ns()), request_id=np.array(request_id),
-                             worker_wall_s=np.float64(time.perf_counter() - started))
+                             worker_wall_s=np.float64(time.perf_counter() - started),
+                             cuda_peak_reserved_bytes=np.int64(cuda_peak_reserved))
                    body = out.getvalue()
                self.send_response(200)
                self.send_header("Content-Type", "application/octet-stream")