[edge-infer] WIP snapshot: uncommitted M0/W1/W4/W5 edge work as of 2026-09-18

Verbatim copy of the edge/omni-edge working tree (tracked diffs + untracked
edge/local, tests/edge/local, docs/edge, pipeline files). Taken so that
vLLM-Omni v0.29.0rc1 can be merged on a branch without touching the
user's working tree. logs_edge/ and benchmark .npy outputs excluded.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Fable 5.1 committed 2026-09-18 09:47:29 +08:00
1 parent 3834de3105
commit 4e3e8a1ebc
62 files changed
+11043 -56

No files matched your search

View File
Whitespace-only changes.
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Fixtures that stand in for the two AMD devices.
Nothing here needs a Radeon, an NPU, onnxruntime or Windows: the worker is
replaced by ``fake_worker.py`` and the devices by hand-built capabilities. The
paths that matter -- framing, the placement gate, the refusal codes -- are
logic, and logic that only runs on one laptop is logic nobody can change
safely.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
from vllm_omni.edge.local.capabilities import (
FORMAT_ONNX_A16W8,
FORMAT_ONNX_FP16,
FORMAT_ONNX_FP32,
DeviceCapability,
)
from vllm_omni.edge.local.external import launch
FAKE_WORKER = str(Path(__file__).resolve().parent / "fake_worker.py")
@pytest.fixture
def fake_route(monkeypatch, tmp_path):
"""Point the ``ort-cpu`` route at the fake worker and this interpreter.
Returns a callable that writes the worker's reply script and hands back the
resolved route.
"""
def configure(**config) -> launch.Route:
config_path = tmp_path / "fake_worker.json"
config_path.write_text(json.dumps(config))
monkeypatch.setenv("VLLM_OMNI_FAKE_WORKER_JSON", str(config_path))
monkeypatch.setenv("VLLM_OMNI_EXTERNAL_WORKER_ORT_CPU", FAKE_WORKER)
monkeypatch.setenv("VLLM_OMNI_EXTERNAL_PYTHON_ORT_CPU", sys.executable)
return launch.resolve(launch.ROUTE_CPU)
return configure
def _device(device_id: str, kind: str, formats, route: str, *, runnable: bool = True,
reason: str = "") -> DeviceCapability:
return DeviceCapability(
device_id=device_id,
kind=kind,
vendor="amd",
name=f"test {kind}",
backend="ort:cpu" if runnable else None,
runnable=runnable,
memory_pool="host_ram",
memory_bytes=32 * 2**30,
compute_capability=None,
weight_formats=frozenset(formats),
evidence="B" if runnable else "D",
reason=reason,
extra={"worker_route": route, "routes": [{"name": route, "available": runnable,
"reason": reason}]},
)
@pytest.fixture
def npu_device() -> DeviceCapability:
"""A16W8 only -- the real constraint, not a simplification."""
return _device("npu:amd", "npu", {FORMAT_ONNX_A16W8}, launch.ROUTE_CPU)
@pytest.fixture
def igpu_device() -> DeviceCapability:
return _device("igpu:amd", "gpu_integrated", {FORMAT_ONNX_FP16, FORMAT_ONNX_FP32},
launch.ROUTE_CPU)
@pytest.fixture
def unreachable_npu() -> DeviceCapability:
return _device(
"npu:amd", "npu", {FORMAT_ONNX_A16W8}, launch.ROUTE_VITISAI,
runnable=False,
reason="an MCDM device; WSL2 forwards display adapters only, so there is no device node here",
)
@pytest.fixture
def a16w8_graph(tmp_path) -> Path:
"""A file that stands in for an exported graph. Never loaded by ORT here."""
path = tmp_path / "stage.onnx"
path.write_bytes(b"not really onnx, and nothing in these tests parses it" * 64)
return path
+110
View File
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""A stage worker that speaks the protocol and never touches a device.
Selected with ``VLLM_OMNI_EXTERNAL_WORKER_ORT_CPU``, so the client, the
framing, the placement gate and every refusal path are exercised on a machine
with no AMD hardware, no onnxruntime and no Windows. What it replies is read
from the JSON file named by ``VLLM_OMNI_FAKE_WORKER_JSON``:
``load`` body merged into the load reply -- set ``node_counts`` and
``fraction_on_target`` to stage a declined graph.
``run`` ``{"scale": float}``; outputs are the inputs times it.
``die_on`` op name after which the worker exits without replying, for the
"worker vanished mid-request" path.
"""
from __future__ import annotations
import importlib.util
import json
import os
import sys
from pathlib import Path
import numpy as np
_HERE = Path(__file__).resolve()
_PROTOCOL = (
_HERE.parents[4] / "vllm_omni" / "edge" / "local" / "external" / "protocol.py"
)
spec = importlib.util.spec_from_file_location("_fake_protocol", _PROTOCOL)
proto = importlib.util.module_from_spec(spec)
sys.modules["_fake_protocol"] = proto
spec.loader.exec_module(proto)
def main() -> int:
host, port, token = sys.argv[2], int(sys.argv[4]), sys.argv[6]
config = {}
config_path = os.environ.get("VLLM_OMNI_FAKE_WORKER_JSON")
if config_path and Path(config_path).is_file():
config = json.loads(Path(config_path).read_text())
import socket
sock = socket.create_connection((host, port), timeout=30)
proto.send_message(
sock,
proto.OP_HELLO,
{
"token": config.get("token", token),
"executable": sys.executable,
"python": sys.version,
"platform": sys.platform,
"pid": os.getpid(),
"onnxruntime": "fake",
"numpy": np.__version__,
"available_providers": ["CPUExecutionProvider"],
"rss_bytes": 1024,
},
)
while True:
op, body, tensors = proto.recv_message(sock)
if config.get("die_on") == op:
sock.close()
return 0
if op == proto.OP_LOAD:
reply = {
"ep": body.get("ep", "cpu"),
"session_providers": ["CPUExecutionProvider"],
"available_providers": ["CPUExecutionProvider"],
"node_counts": {"CPUExecutionProvider": 4},
"total_nodes": 4,
"target_nodes": 4,
"fraction_on_target": 1.0,
"session_create_s": 0.01,
"warmup_s": 0.001,
"rss_bytes": 111 * 2**20,
"inputs": [{"name": n, "type": "tensor(float)", "shape": list(t.shape)}
for n, t in tensors.items()],
"outputs": [{"name": "y", "type": "tensor(float)", "shape": []}],
}
reply.update(config.get("load", {}))
proto.send_message(sock, proto.OP_OK, reply)
elif op == proto.OP_RUN:
scale = float(config.get("run", {}).get("scale", 2.0))
proto.send_message(
sock,
proto.OP_OK,
{"run_s": 0.001},
{f"{k}_out": (v * scale) for k, v in tensors.items()},
)
elif op == proto.OP_STATS:
proto.send_message(
sock, proto.OP_OK,
{"runs": 1, "total_run_s": 0.001, "rss_bytes": 111 * 2**20,
"peak_rss_bytes": 122 * 2**20, "report": {}},
)
elif op == proto.OP_CLOSE:
proto.send_message(sock, proto.OP_OK, {})
return 0
else:
proto.send_message(sock, proto.OP_ERR,
{"message": f"unknown op {op}", "code": "ValueError"})
if __name__ == "__main__":
raise SystemExit(main())
+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Starting, driving and losing a worker process."""
from __future__ import annotations
import numpy as np
import pytest
from vllm_omni.edge.local.external import launch
from vllm_omni.edge.local.external.client import ExternalWorker, WorkerStartError
def test_handshake_and_round_trip(fake_route):
with ExternalWorker(fake_route()) as worker:
assert worker.hello["onnxruntime"] == "fake"
report = worker.load("/nonexistent/stage.onnx", example_inputs={"x": np.ones(4, np.float32)})
assert report.fraction_on_target == 1.0
assert report.rss_bytes == 111 * 2**20
outputs, timing = worker.run({"x": np.ones(4, np.float32)})
assert np.array_equal(outputs["x_out"], np.full(4, 2.0, np.float32))
assert timing.round_trip_s >= timing.worker_s
assert timing.transport_s == pytest.approx(timing.round_trip_s - timing.worker_s)
assert worker.stats()["peak_rss_bytes"] == 122 * 2**20
def test_wrong_token_is_refused(fake_route):
"""The listener is on a host-visible address; a stray peer must not be served."""
worker = ExternalWorker(fake_route(token="not-the-token"))
with pytest.raises(WorkerStartError, match="wrong token"):
worker.start()
worker.close()
def test_worker_that_vanishes_mid_request_reports_rather_than_hangs(fake_route):
worker = ExternalWorker(fake_route(die_on="run"))
worker.start()
worker.load("/nonexistent/stage.onnx", example_inputs={"x": np.ones(2, np.float32)})
with pytest.raises(WorkerStartError, match="worker connection failed"):
worker.run({"x": np.ones(2, np.float32)})
worker.close()
def test_unavailable_route_refuses_at_construction():
route = launch.Route(
name="ort-vitisai", interpreter=None, worker="w.py", ep="vitisai",
is_windows=True, available=False, reason="no interpreter installed",
)
with pytest.raises(WorkerStartError, match="no interpreter installed"):
ExternalWorker(route)
def test_close_is_idempotent(fake_route):
worker = ExternalWorker(fake_route())
worker.start()
worker.close()
worker.close()
def test_bind_address_is_loopback_for_a_same_os_worker():
from vllm_omni.edge.local.external import client
assert client._bind_address(is_windows=False) == "127.0.0.1"
def test_bind_address_for_a_windows_worker_is_not_loopback_under_wsl():
"""The regression this exists for: probing the WSL nameserver instead of the
gateway yields 10.255.255.254, an address only this distro can reach, and
the Windows worker then dies with ECONNREFUSED after a silent start."""
from vllm_omni.edge.local.external import client
if not launch.is_wsl():
pytest.skip("not running under WSL interop")
assert not client._bind_address(is_windows=True).startswith(("127.", "10.255."))
def test_external_data_counts_toward_the_artifact_size(tmp_path):
"""An ONNX graph with external weights is a few MB of topology pointing at
gigabytes beside it. Charging only the .onnx told the admission ledger that
a 1.8 GB vision tower costs 2 MiB."""
import onnx
from onnx import helper, numpy_helper
from vllm_omni.edge.local.manifest import build_graph_artifact
weight = numpy_helper.from_array(np.zeros((256, 256), np.float32), name="w")
node = helper.make_node("MatMul", ["x", "w"], ["y"])
graph = helper.make_graph(
[node], "g",
[helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, [1, 256])],
[helper.make_tensor_value_info("y", onnx.TensorProto.FLOAT, [1, 256])],
[weight],
)
path = tmp_path / "m.onnx"
onnx.save(helper.make_model(graph), str(path), save_as_external_data=True,
all_tensors_to_one_file=True, location="m.onnx.data")
artifact = build_graph_artifact(path, fmt="onnx:fp32", opset=21, source_model="t",
component="c", exporter="e")
assert artifact.bytes > path.stat().st_size
assert artifact.bytes >= 256 * 256 * 4
+94
View File
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The wire format, which has to survive a peer that is not us."""
from __future__ import annotations
import socket
import threading
import numpy as np
import pytest
from vllm_omni.edge.local.external import protocol
def _round_trip(op, body=None, tensors=None):
near, far = socket.socketpair()
try:
threading.Thread(
target=lambda: protocol.send_message(near, op, body, tensors), daemon=True
).start()
return protocol.recv_message(far)
finally:
near.close()
far.close()
def test_tensors_survive_dtype_and_shape():
sent = {
"f32": np.arange(12, dtype=np.float32).reshape(3, 4),
"i64": np.array([[1, 2], [3, 4]], dtype=np.int64),
"u16": np.array([7, 8, 9], dtype=np.uint16),
"scalar": np.array(3.5, dtype=np.float64),
}
op, body, got = _round_trip(protocol.OP_RUN, {"n": 1}, sent)
assert op == protocol.OP_RUN and body == {"n": 1}
for name, array in sent.items():
assert got[name].dtype == array.dtype
assert got[name].shape == array.shape
assert np.array_equal(got[name], array)
def test_received_tensors_are_writable():
"""``frombuffer`` hands back a read-only view aliasing the socket buffer.
ORT's IOBinding writes into the arrays it is given, so a read-only result
would fail at the far end of a working connection rather than here.
"""
_, _, got = _round_trip(protocol.OP_RUN, {}, {"x": np.ones(4, dtype=np.float32)})
got["x"][0] = 5.0 # must not raise
assert got["x"][0] == 5.0
def test_payload_larger_than_one_recv():
"""Stream sockets short-read; a naive recv would misparse the next tensor."""
big = np.arange(4 * 1024 * 1024, dtype=np.float32)
_, _, got = _round_trip(protocol.OP_RUN, {}, {"big": big, "after": np.array([1.0])})
assert np.array_equal(got["big"], big)
assert np.array_equal(got["after"], np.array([1.0]))
def test_empty_message_and_no_tensors():
op, body, got = _round_trip(protocol.OP_STATS)
assert (op, body, got) == (protocol.OP_STATS, {}, {})
def test_error_reply_raises_with_remote_detail():
with pytest.raises(protocol.WorkerError) as excinfo:
_round_trip(protocol.OP_ERR, {"message": "boom", "code": "ValueError",
"traceback": "far side traceback"})
assert excinfo.value.code == "ValueError"
assert "far side traceback" in excinfo.value.remote_traceback
def test_closed_peer_is_a_protocol_error_not_a_hang():
near, far = socket.socketpair()
near.close()
with pytest.raises(protocol.ProtocolError):
protocol.recv_message(far)
far.close()
def test_oversized_header_is_refused_before_allocation():
"""A corrupt length must not become a multi-gigabyte allocation."""
near, far = socket.socketpair()
try:
import struct
near.sendall(struct.pack(">I", protocol.MAX_HEADER_BYTES + 1))
with pytest.raises(protocol.ProtocolError, match="over the limit"):
protocol.recv_message(far)
finally:
near.close()
far.close()
+209
View File
@@ -0,0 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Placing a stage on an external device, and refusing to.
The gate tested here is the one that keeps a CPU fallback from being reported
as NPU execution. It cannot be replaced by a cheaper check: a VitisAI session
lists the EP in ``get_providers()`` whether or not it claimed a node, and a
graph it declined runs on the CPU and returns bit-identical numbers. So these
tests assert on node assignments, never on output values.
"""
from __future__ import annotations
import numpy as np
import pytest
from vllm_omni.edge.local.capabilities import FORMAT_ONNX_A16W8, FORMAT_ONNX_FP16
from vllm_omni.edge.local.external.stage import (
ExternalStage,
PlacementRefused,
plan_external_stage,
verify_placement,
)
from vllm_omni.edge.local.manifest import build_graph_artifact
from vllm_omni.edge.local.plan import (
REFUSE_EP_PLACEMENT,
REFUSE_NO_ARTIFACT,
REFUSE_NO_DEVICE,
REFUSE_ROUTE,
)
def _artifact(path, fmt=FORMAT_ONNX_A16W8, component="stage"):
return build_graph_artifact(
path, fmt=fmt, opset=21, source_model="test", component=component,
exporter="tests.edge.local.external",
)
def test_a16w8_graph_is_placed_on_the_npu(a16w8_graph, npu_device, igpu_device):
plan = plan_external_stage(_artifact(a16w8_graph), [igpu_device, npu_device])
assert plan.admitted
assert plan.device is not None and plan.device.device_id == "npu:amd"
# The iGPU was considered and rejected on format; the record says so rather
# than only naming the winner.
assert [r.code for r in plan.refusals] == [REFUSE_NO_ARTIFACT]
def test_fp16_graph_is_refused_by_the_npu_not_quietly_downgraded(a16w8_graph, npu_device):
"""A16W8 is the only door on XDNA2, and the failure mode for anything else
is silent: no error, every node on the CPU. So it is refused at plan time."""
plan = plan_external_stage(_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device])
assert not plan.admitted
assert plan.refusals[0].code == REFUSE_NO_ARTIFACT
assert "a16w8" in plan.refusals[0].remedy.lower()
def test_require_refuses_rather_than_substituting(a16w8_graph, npu_device, igpu_device):
""""Does the NPU run this" must not be answered with the iGPU."""
plan = plan_external_stage(
_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device, igpu_device],
require="npu:amd",
)
assert not plan.admitted
assert {r.device_id for r in plan.refusals} == {"npu:amd"}
def test_prefer_falls_through(a16w8_graph, npu_device, igpu_device):
plan = plan_external_stage(
_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device, igpu_device],
prefer="npu:amd",
)
assert plan.admitted and plan.device.device_id == "igpu:amd"
def test_unreachable_device_refuses_with_the_route_reason(a16w8_graph, unreachable_npu):
plan = plan_external_stage(_artifact(a16w8_graph), [unreachable_npu])
assert not plan.admitted
assert plan.refusals[0].code == REFUSE_ROUTE
assert "MCDM" in plan.refusals[0].message
assert "VLLM_OMNI_EXTERNAL_PYTHON_ORT_VITISAI" in plan.refusals[0].remedy
def test_no_external_device_at_all(a16w8_graph):
plan = plan_external_stage(_artifact(a16w8_graph), [])
assert not plan.admitted and plan.refusals[0].code == REFUSE_NO_DEVICE
def test_required_device_absent_from_enumeration(a16w8_graph, igpu_device):
plan = plan_external_stage(_artifact(a16w8_graph), [igpu_device], require="npu:amd")
assert not plan.admitted and plan.refusals[0].code == REFUSE_NO_DEVICE
def test_budget_is_on_the_shared_pool(a16w8_graph, npu_device):
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
assert {r.pool for r in plan.reservations} == {"host_ram"}
assert plan.budget_bytes > 0
def test_open_verifies_placement_and_closes_the_ledger(a16w8_graph, npu_device, fake_route, tmp_path):
fake_route()
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
with ExternalStage(plan) as stage:
report = stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
assert report.fraction_on_target == 1.0
runtime = next(r for r in plan.reservations if r.purpose == "worker runtime")
assert runtime.actual_upper_bound_bytes == 111 * 2**20
assert runtime.bytes >= runtime.actual_upper_bound_bytes, "budget must bound the measurement"
def test_declined_graph_is_refused_even_though_it_ran(a16w8_graph, npu_device, fake_route, tmp_path):
"""The whole point: it produced correct output, on the CPU."""
fake_route(load={"node_counts": {"CPUExecutionProvider": 9, "vitisai": 1},
"total_nodes": 10, "target_nodes": 1, "fraction_on_target": 0.1})
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
stage = ExternalStage(plan)
with pytest.raises(PlacementRefused) as excinfo:
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
assert "1 of 10" in excinfo.value.refusal.message
assert "Light" in excinfo.value.refusal.remedy
def test_unverified_placement_is_refused_not_assumed(a16w8_graph, npu_device, fake_route, tmp_path):
"""``None`` means nobody looked. It must not be rounded to "fine"."""
fake_route(load={"fraction_on_target": None, "node_counts": {},
"placement_unknown_reason": "no profiled run"})
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
stage = ExternalStage(plan)
with pytest.raises(PlacementRefused) as excinfo:
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
assert "not measured" in excinfo.value.refusal.message
def test_open_without_example_inputs_is_a_usage_error(a16w8_graph, npu_device, fake_route):
fake_route()
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
stage = ExternalStage(plan)
with pytest.raises(ValueError, match="example_inputs are required"):
stage.open({})
stage.close()
def test_verify_placement_accepts_a_real_split(a16w8_graph, npu_device):
from vllm_omni.edge.local.external.client import LoadReport
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
plan.min_fraction_on_target = 0.3
report = LoadReport(ep="vitisai", graph_path="g.onnx", total_nodes=3, target_nodes=1,
fraction_on_target=1 / 3, node_counts={"vitisai": 1, "CPUExecutionProvider": 2})
assert verify_placement(plan, report) is None
def test_refusing_a_plan_cannot_be_opened(a16w8_graph, npu_device):
plan = plan_external_stage(_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device])
with pytest.raises(PlacementRefused):
ExternalStage(plan)
def test_a_masked_device_is_told_to_drop_the_mask(a16w8_graph, npu_device):
"""The remedy has to name the cause the caller actually has.
A masked device and a device WSL cannot forward are both "unreachable", and
explaining MCDM forwarding to somebody who simply passed ``--mask npu``
sends them after a problem they do not have.
"""
import dataclasses
masked = dataclasses.replace(npu_device, runnable=False, masked=True,
backend=None, reason="masked out by --mask (npu)")
plan = plan_external_stage(_artifact(a16w8_graph), [masked])
assert not plan.admitted
assert plan.refusals[0].code == REFUSE_ROUTE
assert "--mask" in plan.refusals[0].remedy
assert "MCDM" not in plan.refusals[0].remedy
def test_a_stage_larger_than_the_shared_pool_is_refused(a16w8_graph, npu_device):
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device], pool_capacity_bytes=1)
assert not plan.admitted
from vllm_omni.edge.local.plan import REFUSE_CAPACITY
assert plan.refusals[0].code == REFUSE_CAPACITY
assert "same pool the CPU stage is budgeted against" in plan.refusals[0].message
def test_a_silently_swapped_provider_is_refused_by_name(a16w8_graph, npu_device, fake_route, tmp_path):
"""onnxruntime does not raise when a provider fails to initialise.
It prints "Falling back to ['CPUExecutionProvider'] and retrying" and hands
back a working CPU session — which is how a 27B vision tower ran end to end
on the CPU here while the caller believed it was on the iGPU. The report
carries the swap, and it outranks the node count because "the provider
never initialised" says what to fix and "0 of 1171 nodes" does not.
"""
fake_route(load={
"session_providers": ["CPUExecutionProvider"],
"requested_provider_missing": "DmlExecutionProvider",
"fallback_note": "DmlExecutionProvider failed to initialise and onnxruntime silently fell back",
"node_counts": {"CPUExecutionProvider": 113},
"total_nodes": 113, "target_nodes": 0, "fraction_on_target": 0.0,
})
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
stage = ExternalStage(plan)
with pytest.raises(PlacementRefused) as excinfo:
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
assert "silently fell back" in excinfo.value.refusal.message
+64
View File
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shape arithmetic for the exported vision tower.
The tower itself needs a 29 GB checkpoint, so what is covered here is the part
that decides whether an export is even well-formed — and that is where a silent
mistake would land: a grid that does not divide by the merge size produces a
graph that exports, runs, and returns the wrong number of tokens to the LLM.
"""
from __future__ import annotations
import json
import pytest
from vllm_omni.edge.local.artifacts.vision_tower import tower_shape
VISION_CONFIG = {
"depth": 27, "hidden_size": 1152, "in_channels": 3, "intermediate_size": 4304,
"num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 5120,
"patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2,
}
@pytest.fixture
def checkpoint(tmp_path):
(tmp_path / "config.json").write_text(json.dumps({"vision_config": VISION_CONFIG}))
return tmp_path
def test_448_gives_784_patches_and_196_tokens(checkpoint):
shape = tower_shape(checkpoint, 448)
assert (shape.grid_h, shape.grid_w) == (28, 28)
assert shape.patches == 784
# 3 channels x 2 temporal x 16 x 16
assert shape.patch_dim == 1536
# 784 patches merged 2x2
assert shape.merged_tokens == 196
assert shape.out_hidden_size == 5120
def test_image_size_must_divide_by_the_patch_size(checkpoint):
with pytest.raises(ValueError, match="multiple of patch_size"):
tower_shape(checkpoint, 450)
def test_grid_must_divide_by_the_spatial_merge(checkpoint):
"""A 3x3 grid of patches cannot be merged 2x2. Exporting it anyway would
hand the language model a silently wrong number of image tokens."""
with pytest.raises(ValueError, match="spatial_merge_size"):
tower_shape(checkpoint, 48)
def test_a_checkpoint_without_a_tower_is_refused(tmp_path):
(tmp_path / "config.json").write_text(json.dumps({"model_type": "spark2_5"}))
with pytest.raises(ValueError, match="no vision_config"):
tower_shape(tmp_path, 448)
@pytest.mark.parametrize("size,patches,tokens", [(224, 196, 49), (448, 784, 196), (896, 3136, 784)])
def test_token_count_scales_with_area(checkpoint, size, patches, tokens):
shape = tower_shape(checkpoint, size)
assert (shape.patches, shape.merged_tokens) == (patches, tokens)
+106
View File
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""[W4] The native-Windows vLLM route, and why it is refused.
``win-cuda`` is the first route whose far side is torch rather than
onnxruntime, and the first that exists to leave WSL rather than to reach a
device WSL cannot see. Declaring it makes a Windows engine a row in the route
table instead of a parallel universe.
It is also the first route that **cannot pass the placement gate**, and these
tests pin that down so nobody later mistakes the refusal for a bug. M3 made a
measured device attribution a hard admission condition after a VitisAI session
listed its EP, claimed zero nodes, and returned output bit-identical to the
CPU's. vLLM emits no per-operation device map, so a ``win-cuda`` worker has
nothing of that kind to report -- and the rule is about evidence, not about
whose code it is, so our own engine is refused on it exactly as the NPU is.
"""
from __future__ import annotations
import numpy as np
import pytest
from vllm_omni.edge.local.capabilities import FORMAT_ONNX_A16W8
from vllm_omni.edge.local.external import launch
from vllm_omni.edge.local.external.stage import (
ExternalStage,
PlacementRefused,
plan_external_stage,
)
from vllm_omni.edge.local.manifest import build_graph_artifact
from vllm_omni.edge.local.plan import REFUSE_EP_PLACEMENT, REFUSE_NO_DEVICE
from .conftest import _device
def _artifact(path):
return build_graph_artifact(
path, fmt=FORMAT_ONNX_A16W8, opset=21, source_model="test",
component="stage", exporter="tests.edge.local.external",
)
def test_win_cuda_is_a_known_route():
"""It resolves through the same machinery as every other route."""
route = launch.resolve(launch.ROUTE_WIN_CUDA)
assert route.name == "win-cuda"
assert route.ep == "cuda"
# It needs a native-Windows interpreter, like the two AMD Windows routes.
assert route.is_windows is True
# Available or not is environment-dependent; a reason is not.
assert route.available or route.reason
def test_win_cuda_reports_no_placement_evidence():
"""The constant is the contract: ``None`` means nobody can look yet.
Asserted rather than assumed, because the tempting edit -- setting it to
1.0 because the worker "obviously" runs on the GPU -- is exactly the
reasoning M3 disproved.
"""
assert launch.WIN_CUDA_PLACEMENT_EVIDENCE is None
def test_unverified_placement_is_refused_for_our_own_engine(a16w8_graph, fake_route, tmp_path):
"""A worker that cannot say where it ran is refused, not trusted.
The fake worker stands in for a ``win-cuda`` one: it runs, it returns
correct-looking numbers, and it reports ``fraction_on_target=None`` because
vLLM has no per-op device map to derive one from. That is precisely the
shape of a silent CPU fallback, so it must not be admitted.
"""
fake_route(load={
"fraction_on_target": None,
"node_counts": {},
"placement_unknown_reason": "vllm emits no per-operation device map",
})
device = _device("npu:amd", "npu", [FORMAT_ONNX_A16W8], launch.ROUTE_CPU)
plan = plan_external_stage(_artifact(a16w8_graph), [device])
stage = ExternalStage(plan)
with pytest.raises(PlacementRefused) as excinfo:
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
assert "not measured" in excinfo.value.refusal.message
def test_external_planner_does_not_yet_admit_a_discrete_gpu(a16w8_graph, fake_route):
"""A documented gap, pinned so the next person does not rediscover it.
``plan_external_stage`` filters candidates to ``("gpu_integrated", "npu")``
(``stage.py``). Every external route so far reaches a device WSL cannot see
*and* that is integrated or an NPU, so the filter was exactly right. A
``win-cuda`` route breaks that coupling: its device is a **discrete** GPU
reached out-of-process for an OS reason, not a visibility one.
This test asserts the current refusal rather than a desired behaviour. When
the planner learns about externally-routed discrete GPUs, this test should
fail and be rewritten -- that failure is the signal, not a regression.
"""
fake_route()
device = _device("cuda:0", "gpu_discrete", [FORMAT_ONNX_A16W8], launch.ROUTE_CPU)
plan = plan_external_stage(_artifact(a16w8_graph), [device])
assert plan.device is None
assert plan.refusals
assert plan.refusals[0].code == REFUSE_NO_DEVICE
assert "integrated GPU or NPU" in plan.refusals[0].message