[edge-infer] WIP snapshot: uncommitted M0/W1/W4/W5 edge work as of 2026-09-18
Verbatim copy of the edge/omni-edge working tree (tracked diffs + untracked edge/local, tests/edge/local, docs/edge, pipeline files). Taken so that vLLM-Omni v0.29.0rc1 can be merged on a branch without touching the user's working tree. logs_edge/ and benchmark .npy outputs excluded. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
3834de3105
commit
4e3e8a1ebc
62 files changed
+11043
-56
No files matched your search
Whitespace-only changes.
+96
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Fixtures that stand in for the two AMD devices.
|
||||
|
||||
Nothing here needs a Radeon, an NPU, onnxruntime or Windows: the worker is
|
||||
replaced by ``fake_worker.py`` and the devices by hand-built capabilities. The
|
||||
paths that matter -- framing, the placement gate, the refusal codes -- are
|
||||
logic, and logic that only runs on one laptop is logic nobody can change
|
||||
safely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.capabilities import (
|
||||
FORMAT_ONNX_A16W8,
|
||||
FORMAT_ONNX_FP16,
|
||||
FORMAT_ONNX_FP32,
|
||||
DeviceCapability,
|
||||
)
|
||||
from vllm_omni.edge.local.external import launch
|
||||
|
||||
FAKE_WORKER = str(Path(__file__).resolve().parent / "fake_worker.py")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_route(monkeypatch, tmp_path):
|
||||
"""Point the ``ort-cpu`` route at the fake worker and this interpreter.
|
||||
|
||||
Returns a callable that writes the worker's reply script and hands back the
|
||||
resolved route.
|
||||
"""
|
||||
|
||||
def configure(**config) -> launch.Route:
|
||||
config_path = tmp_path / "fake_worker.json"
|
||||
config_path.write_text(json.dumps(config))
|
||||
monkeypatch.setenv("VLLM_OMNI_FAKE_WORKER_JSON", str(config_path))
|
||||
monkeypatch.setenv("VLLM_OMNI_EXTERNAL_WORKER_ORT_CPU", FAKE_WORKER)
|
||||
monkeypatch.setenv("VLLM_OMNI_EXTERNAL_PYTHON_ORT_CPU", sys.executable)
|
||||
return launch.resolve(launch.ROUTE_CPU)
|
||||
|
||||
return configure
|
||||
|
||||
|
||||
def _device(device_id: str, kind: str, formats, route: str, *, runnable: bool = True,
|
||||
reason: str = "") -> DeviceCapability:
|
||||
return DeviceCapability(
|
||||
device_id=device_id,
|
||||
kind=kind,
|
||||
vendor="amd",
|
||||
name=f"test {kind}",
|
||||
backend="ort:cpu" if runnable else None,
|
||||
runnable=runnable,
|
||||
memory_pool="host_ram",
|
||||
memory_bytes=32 * 2**30,
|
||||
compute_capability=None,
|
||||
weight_formats=frozenset(formats),
|
||||
evidence="B" if runnable else "D",
|
||||
reason=reason,
|
||||
extra={"worker_route": route, "routes": [{"name": route, "available": runnable,
|
||||
"reason": reason}]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def npu_device() -> DeviceCapability:
|
||||
"""A16W8 only -- the real constraint, not a simplification."""
|
||||
return _device("npu:amd", "npu", {FORMAT_ONNX_A16W8}, launch.ROUTE_CPU)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def igpu_device() -> DeviceCapability:
|
||||
return _device("igpu:amd", "gpu_integrated", {FORMAT_ONNX_FP16, FORMAT_ONNX_FP32},
|
||||
launch.ROUTE_CPU)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unreachable_npu() -> DeviceCapability:
|
||||
return _device(
|
||||
"npu:amd", "npu", {FORMAT_ONNX_A16W8}, launch.ROUTE_VITISAI,
|
||||
runnable=False,
|
||||
reason="an MCDM device; WSL2 forwards display adapters only, so there is no device node here",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def a16w8_graph(tmp_path) -> Path:
|
||||
"""A file that stands in for an exported graph. Never loaded by ORT here."""
|
||||
path = tmp_path / "stage.onnx"
|
||||
path.write_bytes(b"not really onnx, and nothing in these tests parses it" * 64)
|
||||
return path
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""A stage worker that speaks the protocol and never touches a device.
|
||||
|
||||
Selected with ``VLLM_OMNI_EXTERNAL_WORKER_ORT_CPU``, so the client, the
|
||||
framing, the placement gate and every refusal path are exercised on a machine
|
||||
with no AMD hardware, no onnxruntime and no Windows. What it replies is read
|
||||
from the JSON file named by ``VLLM_OMNI_FAKE_WORKER_JSON``:
|
||||
|
||||
``load`` body merged into the load reply -- set ``node_counts`` and
|
||||
``fraction_on_target`` to stage a declined graph.
|
||||
``run`` ``{"scale": float}``; outputs are the inputs times it.
|
||||
``die_on`` op name after which the worker exits without replying, for the
|
||||
"worker vanished mid-request" path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
_HERE = Path(__file__).resolve()
|
||||
_PROTOCOL = (
|
||||
_HERE.parents[4] / "vllm_omni" / "edge" / "local" / "external" / "protocol.py"
|
||||
)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("_fake_protocol", _PROTOCOL)
|
||||
proto = importlib.util.module_from_spec(spec)
|
||||
sys.modules["_fake_protocol"] = proto
|
||||
spec.loader.exec_module(proto)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
host, port, token = sys.argv[2], int(sys.argv[4]), sys.argv[6]
|
||||
config = {}
|
||||
config_path = os.environ.get("VLLM_OMNI_FAKE_WORKER_JSON")
|
||||
if config_path and Path(config_path).is_file():
|
||||
config = json.loads(Path(config_path).read_text())
|
||||
|
||||
import socket
|
||||
|
||||
sock = socket.create_connection((host, port), timeout=30)
|
||||
proto.send_message(
|
||||
sock,
|
||||
proto.OP_HELLO,
|
||||
{
|
||||
"token": config.get("token", token),
|
||||
"executable": sys.executable,
|
||||
"python": sys.version,
|
||||
"platform": sys.platform,
|
||||
"pid": os.getpid(),
|
||||
"onnxruntime": "fake",
|
||||
"numpy": np.__version__,
|
||||
"available_providers": ["CPUExecutionProvider"],
|
||||
"rss_bytes": 1024,
|
||||
},
|
||||
)
|
||||
|
||||
while True:
|
||||
op, body, tensors = proto.recv_message(sock)
|
||||
if config.get("die_on") == op:
|
||||
sock.close()
|
||||
return 0
|
||||
if op == proto.OP_LOAD:
|
||||
reply = {
|
||||
"ep": body.get("ep", "cpu"),
|
||||
"session_providers": ["CPUExecutionProvider"],
|
||||
"available_providers": ["CPUExecutionProvider"],
|
||||
"node_counts": {"CPUExecutionProvider": 4},
|
||||
"total_nodes": 4,
|
||||
"target_nodes": 4,
|
||||
"fraction_on_target": 1.0,
|
||||
"session_create_s": 0.01,
|
||||
"warmup_s": 0.001,
|
||||
"rss_bytes": 111 * 2**20,
|
||||
"inputs": [{"name": n, "type": "tensor(float)", "shape": list(t.shape)}
|
||||
for n, t in tensors.items()],
|
||||
"outputs": [{"name": "y", "type": "tensor(float)", "shape": []}],
|
||||
}
|
||||
reply.update(config.get("load", {}))
|
||||
proto.send_message(sock, proto.OP_OK, reply)
|
||||
elif op == proto.OP_RUN:
|
||||
scale = float(config.get("run", {}).get("scale", 2.0))
|
||||
proto.send_message(
|
||||
sock,
|
||||
proto.OP_OK,
|
||||
{"run_s": 0.001},
|
||||
{f"{k}_out": (v * scale) for k, v in tensors.items()},
|
||||
)
|
||||
elif op == proto.OP_STATS:
|
||||
proto.send_message(
|
||||
sock, proto.OP_OK,
|
||||
{"runs": 1, "total_run_s": 0.001, "rss_bytes": 111 * 2**20,
|
||||
"peak_rss_bytes": 122 * 2**20, "report": {}},
|
||||
)
|
||||
elif op == proto.OP_CLOSE:
|
||||
proto.send_message(sock, proto.OP_OK, {})
|
||||
return 0
|
||||
else:
|
||||
proto.send_message(sock, proto.OP_ERR,
|
||||
{"message": f"unknown op {op}", "code": "ValueError"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Starting, driving and losing a worker process."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.external import launch
|
||||
from vllm_omni.edge.local.external.client import ExternalWorker, WorkerStartError
|
||||
|
||||
|
||||
def test_handshake_and_round_trip(fake_route):
|
||||
with ExternalWorker(fake_route()) as worker:
|
||||
assert worker.hello["onnxruntime"] == "fake"
|
||||
report = worker.load("/nonexistent/stage.onnx", example_inputs={"x": np.ones(4, np.float32)})
|
||||
assert report.fraction_on_target == 1.0
|
||||
assert report.rss_bytes == 111 * 2**20
|
||||
|
||||
outputs, timing = worker.run({"x": np.ones(4, np.float32)})
|
||||
assert np.array_equal(outputs["x_out"], np.full(4, 2.0, np.float32))
|
||||
assert timing.round_trip_s >= timing.worker_s
|
||||
assert timing.transport_s == pytest.approx(timing.round_trip_s - timing.worker_s)
|
||||
|
||||
assert worker.stats()["peak_rss_bytes"] == 122 * 2**20
|
||||
|
||||
|
||||
def test_wrong_token_is_refused(fake_route):
|
||||
"""The listener is on a host-visible address; a stray peer must not be served."""
|
||||
worker = ExternalWorker(fake_route(token="not-the-token"))
|
||||
with pytest.raises(WorkerStartError, match="wrong token"):
|
||||
worker.start()
|
||||
worker.close()
|
||||
|
||||
|
||||
def test_worker_that_vanishes_mid_request_reports_rather_than_hangs(fake_route):
|
||||
worker = ExternalWorker(fake_route(die_on="run"))
|
||||
worker.start()
|
||||
worker.load("/nonexistent/stage.onnx", example_inputs={"x": np.ones(2, np.float32)})
|
||||
with pytest.raises(WorkerStartError, match="worker connection failed"):
|
||||
worker.run({"x": np.ones(2, np.float32)})
|
||||
worker.close()
|
||||
|
||||
|
||||
def test_unavailable_route_refuses_at_construction():
|
||||
route = launch.Route(
|
||||
name="ort-vitisai", interpreter=None, worker="w.py", ep="vitisai",
|
||||
is_windows=True, available=False, reason="no interpreter installed",
|
||||
)
|
||||
with pytest.raises(WorkerStartError, match="no interpreter installed"):
|
||||
ExternalWorker(route)
|
||||
|
||||
|
||||
def test_close_is_idempotent(fake_route):
|
||||
worker = ExternalWorker(fake_route())
|
||||
worker.start()
|
||||
worker.close()
|
||||
worker.close()
|
||||
|
||||
|
||||
def test_bind_address_is_loopback_for_a_same_os_worker():
|
||||
from vllm_omni.edge.local.external import client
|
||||
|
||||
assert client._bind_address(is_windows=False) == "127.0.0.1"
|
||||
|
||||
|
||||
def test_bind_address_for_a_windows_worker_is_not_loopback_under_wsl():
|
||||
"""The regression this exists for: probing the WSL nameserver instead of the
|
||||
gateway yields 10.255.255.254, an address only this distro can reach, and
|
||||
the Windows worker then dies with ECONNREFUSED after a silent start."""
|
||||
from vllm_omni.edge.local.external import client
|
||||
|
||||
if not launch.is_wsl():
|
||||
pytest.skip("not running under WSL interop")
|
||||
assert not client._bind_address(is_windows=True).startswith(("127.", "10.255."))
|
||||
|
||||
|
||||
def test_external_data_counts_toward_the_artifact_size(tmp_path):
|
||||
"""An ONNX graph with external weights is a few MB of topology pointing at
|
||||
gigabytes beside it. Charging only the .onnx told the admission ledger that
|
||||
a 1.8 GB vision tower costs 2 MiB."""
|
||||
import onnx
|
||||
from onnx import helper, numpy_helper
|
||||
|
||||
from vllm_omni.edge.local.manifest import build_graph_artifact
|
||||
|
||||
weight = numpy_helper.from_array(np.zeros((256, 256), np.float32), name="w")
|
||||
node = helper.make_node("MatMul", ["x", "w"], ["y"])
|
||||
graph = helper.make_graph(
|
||||
[node], "g",
|
||||
[helper.make_tensor_value_info("x", onnx.TensorProto.FLOAT, [1, 256])],
|
||||
[helper.make_tensor_value_info("y", onnx.TensorProto.FLOAT, [1, 256])],
|
||||
[weight],
|
||||
)
|
||||
path = tmp_path / "m.onnx"
|
||||
onnx.save(helper.make_model(graph), str(path), save_as_external_data=True,
|
||||
all_tensors_to_one_file=True, location="m.onnx.data")
|
||||
|
||||
artifact = build_graph_artifact(path, fmt="onnx:fp32", opset=21, source_model="t",
|
||||
component="c", exporter="e")
|
||||
assert artifact.bytes > path.stat().st_size
|
||||
assert artifact.bytes >= 256 * 256 * 4
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The wire format, which has to survive a peer that is not us."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import threading
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.external import protocol
|
||||
|
||||
|
||||
def _round_trip(op, body=None, tensors=None):
|
||||
near, far = socket.socketpair()
|
||||
try:
|
||||
threading.Thread(
|
||||
target=lambda: protocol.send_message(near, op, body, tensors), daemon=True
|
||||
).start()
|
||||
return protocol.recv_message(far)
|
||||
finally:
|
||||
near.close()
|
||||
far.close()
|
||||
|
||||
|
||||
def test_tensors_survive_dtype_and_shape():
|
||||
sent = {
|
||||
"f32": np.arange(12, dtype=np.float32).reshape(3, 4),
|
||||
"i64": np.array([[1, 2], [3, 4]], dtype=np.int64),
|
||||
"u16": np.array([7, 8, 9], dtype=np.uint16),
|
||||
"scalar": np.array(3.5, dtype=np.float64),
|
||||
}
|
||||
op, body, got = _round_trip(protocol.OP_RUN, {"n": 1}, sent)
|
||||
assert op == protocol.OP_RUN and body == {"n": 1}
|
||||
for name, array in sent.items():
|
||||
assert got[name].dtype == array.dtype
|
||||
assert got[name].shape == array.shape
|
||||
assert np.array_equal(got[name], array)
|
||||
|
||||
|
||||
def test_received_tensors_are_writable():
|
||||
"""``frombuffer`` hands back a read-only view aliasing the socket buffer.
|
||||
|
||||
ORT's IOBinding writes into the arrays it is given, so a read-only result
|
||||
would fail at the far end of a working connection rather than here.
|
||||
"""
|
||||
_, _, got = _round_trip(protocol.OP_RUN, {}, {"x": np.ones(4, dtype=np.float32)})
|
||||
got["x"][0] = 5.0 # must not raise
|
||||
assert got["x"][0] == 5.0
|
||||
|
||||
|
||||
def test_payload_larger_than_one_recv():
|
||||
"""Stream sockets short-read; a naive recv would misparse the next tensor."""
|
||||
big = np.arange(4 * 1024 * 1024, dtype=np.float32)
|
||||
_, _, got = _round_trip(protocol.OP_RUN, {}, {"big": big, "after": np.array([1.0])})
|
||||
assert np.array_equal(got["big"], big)
|
||||
assert np.array_equal(got["after"], np.array([1.0]))
|
||||
|
||||
|
||||
def test_empty_message_and_no_tensors():
|
||||
op, body, got = _round_trip(protocol.OP_STATS)
|
||||
assert (op, body, got) == (protocol.OP_STATS, {}, {})
|
||||
|
||||
|
||||
def test_error_reply_raises_with_remote_detail():
|
||||
with pytest.raises(protocol.WorkerError) as excinfo:
|
||||
_round_trip(protocol.OP_ERR, {"message": "boom", "code": "ValueError",
|
||||
"traceback": "far side traceback"})
|
||||
assert excinfo.value.code == "ValueError"
|
||||
assert "far side traceback" in excinfo.value.remote_traceback
|
||||
|
||||
|
||||
def test_closed_peer_is_a_protocol_error_not_a_hang():
|
||||
near, far = socket.socketpair()
|
||||
near.close()
|
||||
with pytest.raises(protocol.ProtocolError):
|
||||
protocol.recv_message(far)
|
||||
far.close()
|
||||
|
||||
|
||||
def test_oversized_header_is_refused_before_allocation():
|
||||
"""A corrupt length must not become a multi-gigabyte allocation."""
|
||||
near, far = socket.socketpair()
|
||||
try:
|
||||
import struct
|
||||
|
||||
near.sendall(struct.pack(">I", protocol.MAX_HEADER_BYTES + 1))
|
||||
with pytest.raises(protocol.ProtocolError, match="over the limit"):
|
||||
protocol.recv_message(far)
|
||||
finally:
|
||||
near.close()
|
||||
far.close()
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Placing a stage on an external device, and refusing to.
|
||||
|
||||
The gate tested here is the one that keeps a CPU fallback from being reported
|
||||
as NPU execution. It cannot be replaced by a cheaper check: a VitisAI session
|
||||
lists the EP in ``get_providers()`` whether or not it claimed a node, and a
|
||||
graph it declined runs on the CPU and returns bit-identical numbers. So these
|
||||
tests assert on node assignments, never on output values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.capabilities import FORMAT_ONNX_A16W8, FORMAT_ONNX_FP16
|
||||
from vllm_omni.edge.local.external.stage import (
|
||||
ExternalStage,
|
||||
PlacementRefused,
|
||||
plan_external_stage,
|
||||
verify_placement,
|
||||
)
|
||||
from vllm_omni.edge.local.manifest import build_graph_artifact
|
||||
from vllm_omni.edge.local.plan import (
|
||||
REFUSE_EP_PLACEMENT,
|
||||
REFUSE_NO_ARTIFACT,
|
||||
REFUSE_NO_DEVICE,
|
||||
REFUSE_ROUTE,
|
||||
)
|
||||
|
||||
|
||||
def _artifact(path, fmt=FORMAT_ONNX_A16W8, component="stage"):
|
||||
return build_graph_artifact(
|
||||
path, fmt=fmt, opset=21, source_model="test", component=component,
|
||||
exporter="tests.edge.local.external",
|
||||
)
|
||||
|
||||
|
||||
def test_a16w8_graph_is_placed_on_the_npu(a16w8_graph, npu_device, igpu_device):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [igpu_device, npu_device])
|
||||
assert plan.admitted
|
||||
assert plan.device is not None and plan.device.device_id == "npu:amd"
|
||||
# The iGPU was considered and rejected on format; the record says so rather
|
||||
# than only naming the winner.
|
||||
assert [r.code for r in plan.refusals] == [REFUSE_NO_ARTIFACT]
|
||||
|
||||
|
||||
def test_fp16_graph_is_refused_by_the_npu_not_quietly_downgraded(a16w8_graph, npu_device):
|
||||
"""A16W8 is the only door on XDNA2, and the failure mode for anything else
|
||||
is silent: no error, every node on the CPU. So it is refused at plan time."""
|
||||
plan = plan_external_stage(_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device])
|
||||
assert not plan.admitted
|
||||
assert plan.refusals[0].code == REFUSE_NO_ARTIFACT
|
||||
assert "a16w8" in plan.refusals[0].remedy.lower()
|
||||
|
||||
|
||||
def test_require_refuses_rather_than_substituting(a16w8_graph, npu_device, igpu_device):
|
||||
""""Does the NPU run this" must not be answered with the iGPU."""
|
||||
plan = plan_external_stage(
|
||||
_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device, igpu_device],
|
||||
require="npu:amd",
|
||||
)
|
||||
assert not plan.admitted
|
||||
assert {r.device_id for r in plan.refusals} == {"npu:amd"}
|
||||
|
||||
|
||||
def test_prefer_falls_through(a16w8_graph, npu_device, igpu_device):
|
||||
plan = plan_external_stage(
|
||||
_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device, igpu_device],
|
||||
prefer="npu:amd",
|
||||
)
|
||||
assert plan.admitted and plan.device.device_id == "igpu:amd"
|
||||
|
||||
|
||||
def test_unreachable_device_refuses_with_the_route_reason(a16w8_graph, unreachable_npu):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [unreachable_npu])
|
||||
assert not plan.admitted
|
||||
assert plan.refusals[0].code == REFUSE_ROUTE
|
||||
assert "MCDM" in plan.refusals[0].message
|
||||
assert "VLLM_OMNI_EXTERNAL_PYTHON_ORT_VITISAI" in plan.refusals[0].remedy
|
||||
|
||||
|
||||
def test_no_external_device_at_all(a16w8_graph):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [])
|
||||
assert not plan.admitted and plan.refusals[0].code == REFUSE_NO_DEVICE
|
||||
|
||||
|
||||
def test_required_device_absent_from_enumeration(a16w8_graph, igpu_device):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [igpu_device], require="npu:amd")
|
||||
assert not plan.admitted and plan.refusals[0].code == REFUSE_NO_DEVICE
|
||||
|
||||
|
||||
def test_budget_is_on_the_shared_pool(a16w8_graph, npu_device):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
assert {r.pool for r in plan.reservations} == {"host_ram"}
|
||||
assert plan.budget_bytes > 0
|
||||
|
||||
|
||||
def test_open_verifies_placement_and_closes_the_ledger(a16w8_graph, npu_device, fake_route, tmp_path):
|
||||
fake_route()
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
with ExternalStage(plan) as stage:
|
||||
report = stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
|
||||
assert report.fraction_on_target == 1.0
|
||||
runtime = next(r for r in plan.reservations if r.purpose == "worker runtime")
|
||||
assert runtime.actual_upper_bound_bytes == 111 * 2**20
|
||||
assert runtime.bytes >= runtime.actual_upper_bound_bytes, "budget must bound the measurement"
|
||||
|
||||
|
||||
def test_declined_graph_is_refused_even_though_it_ran(a16w8_graph, npu_device, fake_route, tmp_path):
|
||||
"""The whole point: it produced correct output, on the CPU."""
|
||||
fake_route(load={"node_counts": {"CPUExecutionProvider": 9, "vitisai": 1},
|
||||
"total_nodes": 10, "target_nodes": 1, "fraction_on_target": 0.1})
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
stage = ExternalStage(plan)
|
||||
with pytest.raises(PlacementRefused) as excinfo:
|
||||
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
|
||||
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
|
||||
assert "1 of 10" in excinfo.value.refusal.message
|
||||
assert "Light" in excinfo.value.refusal.remedy
|
||||
|
||||
|
||||
def test_unverified_placement_is_refused_not_assumed(a16w8_graph, npu_device, fake_route, tmp_path):
|
||||
"""``None`` means nobody looked. It must not be rounded to "fine"."""
|
||||
fake_route(load={"fraction_on_target": None, "node_counts": {},
|
||||
"placement_unknown_reason": "no profiled run"})
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
stage = ExternalStage(plan)
|
||||
with pytest.raises(PlacementRefused) as excinfo:
|
||||
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
|
||||
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
|
||||
assert "not measured" in excinfo.value.refusal.message
|
||||
|
||||
|
||||
def test_open_without_example_inputs_is_a_usage_error(a16w8_graph, npu_device, fake_route):
|
||||
fake_route()
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
stage = ExternalStage(plan)
|
||||
with pytest.raises(ValueError, match="example_inputs are required"):
|
||||
stage.open({})
|
||||
stage.close()
|
||||
|
||||
|
||||
def test_verify_placement_accepts_a_real_split(a16w8_graph, npu_device):
|
||||
from vllm_omni.edge.local.external.client import LoadReport
|
||||
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
plan.min_fraction_on_target = 0.3
|
||||
report = LoadReport(ep="vitisai", graph_path="g.onnx", total_nodes=3, target_nodes=1,
|
||||
fraction_on_target=1 / 3, node_counts={"vitisai": 1, "CPUExecutionProvider": 2})
|
||||
assert verify_placement(plan, report) is None
|
||||
|
||||
|
||||
def test_refusing_a_plan_cannot_be_opened(a16w8_graph, npu_device):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph, fmt=FORMAT_ONNX_FP16), [npu_device])
|
||||
with pytest.raises(PlacementRefused):
|
||||
ExternalStage(plan)
|
||||
|
||||
|
||||
def test_a_masked_device_is_told_to_drop_the_mask(a16w8_graph, npu_device):
|
||||
"""The remedy has to name the cause the caller actually has.
|
||||
|
||||
A masked device and a device WSL cannot forward are both "unreachable", and
|
||||
explaining MCDM forwarding to somebody who simply passed ``--mask npu``
|
||||
sends them after a problem they do not have.
|
||||
"""
|
||||
import dataclasses
|
||||
|
||||
masked = dataclasses.replace(npu_device, runnable=False, masked=True,
|
||||
backend=None, reason="masked out by --mask (npu)")
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [masked])
|
||||
assert not plan.admitted
|
||||
assert plan.refusals[0].code == REFUSE_ROUTE
|
||||
assert "--mask" in plan.refusals[0].remedy
|
||||
assert "MCDM" not in plan.refusals[0].remedy
|
||||
|
||||
|
||||
def test_a_stage_larger_than_the_shared_pool_is_refused(a16w8_graph, npu_device):
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device], pool_capacity_bytes=1)
|
||||
assert not plan.admitted
|
||||
from vllm_omni.edge.local.plan import REFUSE_CAPACITY
|
||||
|
||||
assert plan.refusals[0].code == REFUSE_CAPACITY
|
||||
assert "same pool the CPU stage is budgeted against" in plan.refusals[0].message
|
||||
|
||||
|
||||
def test_a_silently_swapped_provider_is_refused_by_name(a16w8_graph, npu_device, fake_route, tmp_path):
|
||||
"""onnxruntime does not raise when a provider fails to initialise.
|
||||
|
||||
It prints "Falling back to ['CPUExecutionProvider'] and retrying" and hands
|
||||
back a working CPU session — which is how a 27B vision tower ran end to end
|
||||
on the CPU here while the caller believed it was on the iGPU. The report
|
||||
carries the swap, and it outranks the node count because "the provider
|
||||
never initialised" says what to fix and "0 of 1171 nodes" does not.
|
||||
"""
|
||||
fake_route(load={
|
||||
"session_providers": ["CPUExecutionProvider"],
|
||||
"requested_provider_missing": "DmlExecutionProvider",
|
||||
"fallback_note": "DmlExecutionProvider failed to initialise and onnxruntime silently fell back",
|
||||
"node_counts": {"CPUExecutionProvider": 113},
|
||||
"total_nodes": 113, "target_nodes": 0, "fraction_on_target": 0.0,
|
||||
})
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [npu_device])
|
||||
stage = ExternalStage(plan)
|
||||
with pytest.raises(PlacementRefused) as excinfo:
|
||||
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
|
||||
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
|
||||
assert "silently fell back" in excinfo.value.refusal.message
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shape arithmetic for the exported vision tower.
|
||||
|
||||
The tower itself needs a 29 GB checkpoint, so what is covered here is the part
|
||||
that decides whether an export is even well-formed — and that is where a silent
|
||||
mistake would land: a grid that does not divide by the merge size produces a
|
||||
graph that exports, runs, and returns the wrong number of tokens to the LLM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.artifacts.vision_tower import tower_shape
|
||||
|
||||
VISION_CONFIG = {
|
||||
"depth": 27, "hidden_size": 1152, "in_channels": 3, "intermediate_size": 4304,
|
||||
"num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 5120,
|
||||
"patch_size": 16, "spatial_merge_size": 2, "temporal_patch_size": 2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def checkpoint(tmp_path):
|
||||
(tmp_path / "config.json").write_text(json.dumps({"vision_config": VISION_CONFIG}))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_448_gives_784_patches_and_196_tokens(checkpoint):
|
||||
shape = tower_shape(checkpoint, 448)
|
||||
assert (shape.grid_h, shape.grid_w) == (28, 28)
|
||||
assert shape.patches == 784
|
||||
# 3 channels x 2 temporal x 16 x 16
|
||||
assert shape.patch_dim == 1536
|
||||
# 784 patches merged 2x2
|
||||
assert shape.merged_tokens == 196
|
||||
assert shape.out_hidden_size == 5120
|
||||
|
||||
|
||||
def test_image_size_must_divide_by_the_patch_size(checkpoint):
|
||||
with pytest.raises(ValueError, match="multiple of patch_size"):
|
||||
tower_shape(checkpoint, 450)
|
||||
|
||||
|
||||
def test_grid_must_divide_by_the_spatial_merge(checkpoint):
|
||||
"""A 3x3 grid of patches cannot be merged 2x2. Exporting it anyway would
|
||||
hand the language model a silently wrong number of image tokens."""
|
||||
with pytest.raises(ValueError, match="spatial_merge_size"):
|
||||
tower_shape(checkpoint, 48)
|
||||
|
||||
|
||||
def test_a_checkpoint_without_a_tower_is_refused(tmp_path):
|
||||
(tmp_path / "config.json").write_text(json.dumps({"model_type": "spark2_5"}))
|
||||
with pytest.raises(ValueError, match="no vision_config"):
|
||||
tower_shape(tmp_path, 448)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size,patches,tokens", [(224, 196, 49), (448, 784, 196), (896, 3136, 784)])
|
||||
def test_token_count_scales_with_area(checkpoint, size, patches, tokens):
|
||||
shape = tower_shape(checkpoint, size)
|
||||
assert (shape.patches, shape.merged_tokens) == (patches, tokens)
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""[W4] The native-Windows vLLM route, and why it is refused.
|
||||
|
||||
``win-cuda`` is the first route whose far side is torch rather than
|
||||
onnxruntime, and the first that exists to leave WSL rather than to reach a
|
||||
device WSL cannot see. Declaring it makes a Windows engine a row in the route
|
||||
table instead of a parallel universe.
|
||||
|
||||
It is also the first route that **cannot pass the placement gate**, and these
|
||||
tests pin that down so nobody later mistakes the refusal for a bug. M3 made a
|
||||
measured device attribution a hard admission condition after a VitisAI session
|
||||
listed its EP, claimed zero nodes, and returned output bit-identical to the
|
||||
CPU's. vLLM emits no per-operation device map, so a ``win-cuda`` worker has
|
||||
nothing of that kind to report -- and the rule is about evidence, not about
|
||||
whose code it is, so our own engine is refused on it exactly as the NPU is.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge.local.capabilities import FORMAT_ONNX_A16W8
|
||||
from vllm_omni.edge.local.external import launch
|
||||
from vllm_omni.edge.local.external.stage import (
|
||||
ExternalStage,
|
||||
PlacementRefused,
|
||||
plan_external_stage,
|
||||
)
|
||||
from vllm_omni.edge.local.manifest import build_graph_artifact
|
||||
from vllm_omni.edge.local.plan import REFUSE_EP_PLACEMENT, REFUSE_NO_DEVICE
|
||||
|
||||
from .conftest import _device
|
||||
|
||||
|
||||
def _artifact(path):
|
||||
return build_graph_artifact(
|
||||
path, fmt=FORMAT_ONNX_A16W8, opset=21, source_model="test",
|
||||
component="stage", exporter="tests.edge.local.external",
|
||||
)
|
||||
|
||||
|
||||
def test_win_cuda_is_a_known_route():
|
||||
"""It resolves through the same machinery as every other route."""
|
||||
route = launch.resolve(launch.ROUTE_WIN_CUDA)
|
||||
assert route.name == "win-cuda"
|
||||
assert route.ep == "cuda"
|
||||
# It needs a native-Windows interpreter, like the two AMD Windows routes.
|
||||
assert route.is_windows is True
|
||||
# Available or not is environment-dependent; a reason is not.
|
||||
assert route.available or route.reason
|
||||
|
||||
|
||||
def test_win_cuda_reports_no_placement_evidence():
|
||||
"""The constant is the contract: ``None`` means nobody can look yet.
|
||||
|
||||
Asserted rather than assumed, because the tempting edit -- setting it to
|
||||
1.0 because the worker "obviously" runs on the GPU -- is exactly the
|
||||
reasoning M3 disproved.
|
||||
"""
|
||||
assert launch.WIN_CUDA_PLACEMENT_EVIDENCE is None
|
||||
|
||||
|
||||
def test_unverified_placement_is_refused_for_our_own_engine(a16w8_graph, fake_route, tmp_path):
|
||||
"""A worker that cannot say where it ran is refused, not trusted.
|
||||
|
||||
The fake worker stands in for a ``win-cuda`` one: it runs, it returns
|
||||
correct-looking numbers, and it reports ``fraction_on_target=None`` because
|
||||
vLLM has no per-op device map to derive one from. That is precisely the
|
||||
shape of a silent CPU fallback, so it must not be admitted.
|
||||
"""
|
||||
fake_route(load={
|
||||
"fraction_on_target": None,
|
||||
"node_counts": {},
|
||||
"placement_unknown_reason": "vllm emits no per-operation device map",
|
||||
})
|
||||
device = _device("npu:amd", "npu", [FORMAT_ONNX_A16W8], launch.ROUTE_CPU)
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [device])
|
||||
stage = ExternalStage(plan)
|
||||
with pytest.raises(PlacementRefused) as excinfo:
|
||||
stage.open({"x": np.ones(4, np.float32)}, profile_dir=tmp_path)
|
||||
assert excinfo.value.refusal.code == REFUSE_EP_PLACEMENT
|
||||
assert "not measured" in excinfo.value.refusal.message
|
||||
|
||||
|
||||
def test_external_planner_does_not_yet_admit_a_discrete_gpu(a16w8_graph, fake_route):
|
||||
"""A documented gap, pinned so the next person does not rediscover it.
|
||||
|
||||
``plan_external_stage`` filters candidates to ``("gpu_integrated", "npu")``
|
||||
(``stage.py``). Every external route so far reaches a device WSL cannot see
|
||||
*and* that is integrated or an NPU, so the filter was exactly right. A
|
||||
``win-cuda`` route breaks that coupling: its device is a **discrete** GPU
|
||||
reached out-of-process for an OS reason, not a visibility one.
|
||||
|
||||
This test asserts the current refusal rather than a desired behaviour. When
|
||||
the planner learns about externally-routed discrete GPUs, this test should
|
||||
fail and be rewritten -- that failure is the signal, not a regression.
|
||||
"""
|
||||
fake_route()
|
||||
device = _device("cuda:0", "gpu_discrete", [FORMAT_ONNX_A16W8], launch.ROUTE_CPU)
|
||||
plan = plan_external_stage(_artifact(a16w8_graph), [device])
|
||||
assert plan.device is None
|
||||
assert plan.refusals
|
||||
assert plan.refusals[0].code == REFUSE_NO_DEVICE
|
||||
assert "integrated GPU or NPU" in plan.refusals[0].message
|
||||
Reference in new issue
Block a user