Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
380127049a | ||
|
|
743953b821 | ||
|
|
86166fd988 |
No files matched your search
@@ -0,0 +1,57 @@
|
||||
# Host weight dependency memory diagnostic
|
||||
|
||||
`safetensors_retention.py` isolates repeated CPU `get_tensor()` calls from
|
||||
access to one cached tensor view. It imports neither vLLM nor HWR and uses a
|
||||
synthetic float32 payload controlled by `--tensor-elements` (default: 64).
|
||||
Tensor size can affect retention: a negative result for 16 elements does not
|
||||
rule out growth for 64 elements. The selected size is recorded in the JSON
|
||||
arguments and used for both file creation and the correctness check.
|
||||
HWR itself calls `get_tensor()` when
|
||||
acquiring a lease, then exposes cached views; this probe does not measure HWR
|
||||
requests or repeated lease acquisition.
|
||||
|
||||
Run one mode per fresh process in the same Python environment and CPU affinity:
|
||||
|
||||
```bash
|
||||
git rev-parse HEAD > revision.txt
|
||||
git status --short > working-tree.txt
|
||||
git diff > working-tree.patch
|
||||
# Select an allowed CPU from your process affinity; 0 is only an example.
|
||||
CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
|
||||
benchmarks/host_weight_runtime/safetensors_retention.py \
|
||||
--mode get_tensor --iterations 10000 --sample-every 5000 > feasibility.json
|
||||
```
|
||||
|
||||
For a size comparison, run 16 and 64 elements twice each in fresh processes,
|
||||
with cached reuse as a control at both sizes:
|
||||
|
||||
```bash
|
||||
for repetition in 1 2; do
|
||||
for elements in 16 64; do
|
||||
for mode in get_tensor reuse; do
|
||||
CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
|
||||
benchmarks/host_weight_runtime/safetensors_retention.py \
|
||||
--mode "$mode" --tensor-elements "$elements" \
|
||||
> "$mode-$elements-$repetition.json"
|
||||
done
|
||||
done
|
||||
done
|
||||
```
|
||||
|
||||
The diagnostic sets Torch to one CPU thread, warms up before the baseline,
|
||||
and reports preparation separately from timed loop work. Checkpoints run GC
|
||||
and report Linux private/anonymous memory, RSS, and open descriptors. It checks
|
||||
that transient tensor objects and the final cached view are released. Linux
|
||||
`/proc/self/smaps_rollup` must be readable; invalid arguments and missing proc
|
||||
support fail explicitly. Keep raw JSON and the repository snapshot with results.
|
||||
|
||||
Compare the final loop sample with the warmed baseline, then inspect the closed
|
||||
sample. Private-memory growth after Python tensor objects disappear is evidence
|
||||
of retention in the dependency/allocator path, not proof of a leak's root cause.
|
||||
RSS can include file-backed pages; it is not a private-memory metric. Samples
|
||||
also include small diagnostic bookkeeping allocations. Do not assert a portable
|
||||
memory threshold or turn this synthetic loop into a per-request service estimate.
|
||||
|
||||
Do not drop shared caches, trim allocators, or add serving-time GC to make the
|
||||
numbers look smaller. A dependency replacement or version change requires its
|
||||
own reproducible comparison and correctness/lifecycle validation.
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Linux CPU diagnostic: repeated safetensors materialization versus view reuse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import tempfile
|
||||
import time
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
parsed = int(value)
|
||||
if parsed <= 0:
|
||||
raise argparse.ArgumentTypeError("must be positive")
|
||||
return parsed
|
||||
|
||||
|
||||
def sample(phase: str, iterations: int, loop_seconds: float) -> dict[str, object]:
|
||||
gc.collect()
|
||||
counters = {}
|
||||
for line in Path("/proc/self/smaps_rollup").read_text().splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) == 3 and fields[2] == "kB":
|
||||
counters[fields[0].removesuffix(":")] = int(fields[1]) * 1024
|
||||
return {
|
||||
"phase": phase,
|
||||
"iterations": iterations,
|
||||
"loop_seconds": loop_seconds,
|
||||
"rss_bytes": counters["Rss"],
|
||||
"private_bytes": counters["Private_Clean"] + counters["Private_Dirty"],
|
||||
"anonymous_bytes": counters["Anonymous"],
|
||||
"fd_count": len(tuple(Path("/proc/self/fd").iterdir())),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mode", choices=("get_tensor", "reuse"), required=True)
|
||||
parser.add_argument(
|
||||
"--tensor-elements", type=positive_int, default=64, help="Number of float32 elements in the synthetic tensor"
|
||||
)
|
||||
parser.add_argument("--iterations", type=positive_int, default=1_000_000)
|
||||
parser.add_argument("--sample-every", type=positive_int, default=100_000)
|
||||
parser.add_argument("--warmup", type=positive_int, default=1000)
|
||||
args = parser.parse_args()
|
||||
if not Path("/proc/self/smaps_rollup").is_file():
|
||||
parser.error("requires Linux /proc/self/smaps_rollup")
|
||||
|
||||
preparation_start = time.monotonic()
|
||||
import safetensors
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
samples = [sample("before_file", 0, 0.0)]
|
||||
with tempfile.TemporaryDirectory(prefix="safetensors-retention-") as directory:
|
||||
path = Path(directory) / "synthetic.safetensors"
|
||||
save_file({"weight": torch.arange(args.tensor_elements, dtype=torch.float32)}, str(path))
|
||||
with safe_open(path, framework="pt", device="cpu") as reader:
|
||||
cached = reader.get_tensor("weight")
|
||||
assert torch.equal(cached, torch.arange(args.tensor_elements, dtype=torch.float32))
|
||||
for _ in range(args.warmup):
|
||||
tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
|
||||
del tensor
|
||||
samples.append(sample("baseline", 0, 0.0))
|
||||
preparation_seconds = time.monotonic() - preparation_start
|
||||
completed = 0
|
||||
loop_seconds = 0.0
|
||||
while completed < args.iterations:
|
||||
count = min(args.sample_every, args.iterations - completed)
|
||||
start = time.monotonic()
|
||||
for _ in range(count):
|
||||
tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
|
||||
last_tensor = weakref.ref(tensor)
|
||||
del tensor
|
||||
loop_seconds += time.monotonic() - start
|
||||
completed += count
|
||||
snapshot = sample("loop", completed, loop_seconds)
|
||||
snapshot["last_tensor_alive"] = last_tensor() is not None
|
||||
if args.mode == "get_tensor":
|
||||
assert last_tensor() is None, "diagnostic retained a transient tensor"
|
||||
samples.append(snapshot)
|
||||
del cached
|
||||
del reader
|
||||
samples.append(sample("closed", completed, loop_seconds))
|
||||
assert last_tensor() is None, "last tensor survived reader/view teardown"
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"arguments": vars(args),
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"safetensors": safetensors.__version__,
|
||||
"kernel": platform.release(),
|
||||
"hostname": platform.node(),
|
||||
"pid": os.getpid(),
|
||||
"cpu_affinity": sorted(os.sched_getaffinity(0)),
|
||||
"torch_threads": torch.get_num_threads(),
|
||||
"pythonmalloc": os.environ.get("PYTHONMALLOC", "default"),
|
||||
"preparation_seconds": preparation_seconds,
|
||||
"samples": samples,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -42,6 +42,11 @@ V1 does not include:
|
||||
|
||||
## Motivation and use cases
|
||||
|
||||
For investigating CPU memory retained by dependency tensor materialization,
|
||||
see the [standalone safetensors diagnostic](https://github.com/vllm-project/vllm-omni/blob/main/benchmarks/host_weight_runtime/README.md).
|
||||
It distinguishes repeated dependency calls from reuse of cached views and does
|
||||
not establish per-request HWR leakage.
|
||||
|
||||
Model loading can create the same final host representation repeatedly. This
|
||||
is especially expensive when loading performs checkpoint decoding, tensor
|
||||
renaming, TP slicing, quantization, packing, or scale construction before GPU
|
||||
@@ -313,16 +318,6 @@ One process per exact identity owns a build; other workers wait and then acquire
|
||||
leases for the published artifact. Publication is invisible until all payloads
|
||||
and metadata are validated, hashed, fsynced, and atomically renamed.
|
||||
|
||||
By default, different identities may build concurrently. A domain configured
|
||||
with `CapacityPolicy(build_admission="serialized", max_store_bytes=...)` admits
|
||||
only one producer/publication operation at a time. Warm hits remain concurrent,
|
||||
and admission uses the existing coordination deadline. All workers must agree
|
||||
on the serialized domain's schema-2 capacity policy; use a new root when opting
|
||||
in, since in-place migration of an active domain is unsupported. This prevents
|
||||
competing producers from racing allocation checks but does not provide a
|
||||
filesystem-wide byte quota or automatic stale-data reclamation. See the
|
||||
[module capacity contract](../module/host_weight_runtime.md#validation-and-capacity).
|
||||
|
||||
`coordination_timeout_seconds` bounds filesystem lock acquisition. It does not
|
||||
cancel synchronous validation, a producer that has already started, or atomic
|
||||
publication. A hung in-process producer therefore blocks its owning process and
|
||||
|
||||
@@ -311,41 +311,9 @@ point-in-time evidence only; kernel locks are not a persistent owner registry.
|
||||
Capacity policy covers all store-owned bytes, including ready, temporary, and
|
||||
quarantined data. The local writer preflights `max_artifact_bytes`,
|
||||
`max_store_bytes`, and `min_free_bytes`, and preallocates payloads where the
|
||||
filesystem supports it. `ENOSPC` remains a normal typed store failure.
|
||||
|
||||
`CapacityPolicy.build_admission` defaults to `concurrent`, preserving parallel
|
||||
production for different identities and best-effort capacity preflight. An
|
||||
explicit `serialized` policy, which requires `max_store_bytes`, admits one
|
||||
cooperative build/publication operation per domain through
|
||||
`locks/domain-build.lock`. Production lock order becomes domain admission,
|
||||
per-key build, then artifact. Validated warm hits bypass admission; queued
|
||||
same-key callers still recheck and join a completed artifact. Admission waiting
|
||||
uses the caller's existing coordination deadline and returns a retryable build
|
||||
timeout without interrupting its owner.
|
||||
|
||||
The kernel releases admission on process exit. An explicit same-key retry uses
|
||||
the existing stale-temp recovery; files from unrelated dead builds continue to
|
||||
count toward capacity. A synchronous producer that hangs still requires
|
||||
external process supervision. Admission release errors are logged without
|
||||
revising an already determined publication result.
|
||||
|
||||
Concurrent domains retain their exact schema-1 capacity document. Serialized
|
||||
domains use a schema-2 capacity document with `build_admission=serialized`, so
|
||||
mixed policies and old readers fail initialization rather than bypass admission.
|
||||
Opt into serialization using a new root with an agreed policy; in-place policy
|
||||
migration while old workers may exist is unsupported. For example:
|
||||
|
||||
```python
|
||||
from vllm_omni.host_weight_runtime import CapacityPolicy
|
||||
|
||||
capacity = CapacityPolicy(max_store_bytes=192 * 1024**3, build_admission="serialized")
|
||||
```
|
||||
|
||||
Serialization prevents cooperative producers from racing each other's
|
||||
allocation checks. It is not a filesystem quota: control metadata, finalization,
|
||||
and outside writers can still affect total usage. Automatic eviction and
|
||||
parallel byte reservations are not implemented. This policy trades parallel
|
||||
cold-build throughput for predictable producer admission.
|
||||
filesystem supports it. `ENOSPC` remains a normal typed store failure because
|
||||
concurrent preflight is inherently racy. Automatic eviction and strict
|
||||
concurrent reservations are not implemented.
|
||||
|
||||
For tmpfs, artifact bytes are host-memory consumption and may consume swap;
|
||||
they must not be reported operationally as ordinary disk capacity. The store
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU process and policy coverage for domain-wide producer admission."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import json
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import time
|
||||
from multiprocessing.connection import Connection
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.host_weight_runtime.test_filesystem_store import FakeProducer, _identity, _make_store, _publish_test_artifact
|
||||
from vllm_omni.host_weight_runtime import (
|
||||
BuildRequest,
|
||||
CapacityPolicy,
|
||||
FailureCode,
|
||||
HostWeightError,
|
||||
HostWeightRuntime,
|
||||
HostWeightRuntimeConfig,
|
||||
ProductionMetadata,
|
||||
ResolutionOutcome,
|
||||
RuntimeMode,
|
||||
StoreResult,
|
||||
StoreStatus,
|
||||
TensorWriteSpec,
|
||||
ValidationLevel,
|
||||
WaitPolicy,
|
||||
WeightArtifactIdentity,
|
||||
WeightProductionSpec,
|
||||
)
|
||||
from vllm_omni.host_weight_runtime.filesystem.locks import FileLock, lock_is_active
|
||||
from vllm_omni.host_weight_runtime.protocols import ArtifactWriter
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class AdmissionProducer:
|
||||
def __init__(self, identity: WeightArtifactIdentity, control: Connection | None = None) -> None:
|
||||
self._spec = FakeProducer(identity).spec
|
||||
self.control = control
|
||||
|
||||
@property
|
||||
def spec(self) -> WeightProductionSpec:
|
||||
return self._spec
|
||||
|
||||
def produce(self, writer: ArtifactWriter) -> ProductionMetadata:
|
||||
action = "produce"
|
||||
if self.control is not None:
|
||||
self.control.send("entered")
|
||||
if not self.control.poll(60):
|
||||
raise TimeoutError("test did not release producer")
|
||||
action = self.control.recv()
|
||||
spec = TensorWriteSpec("payload", (65536,), torch.bfloat16)
|
||||
with writer.open_tensor_file("weights.safetensors", (spec,)) as output:
|
||||
output.write_tensor("payload", torch.arange(65536, dtype=torch.float32).to(torch.bfloat16))
|
||||
if action == "crash":
|
||||
os._exit(27)
|
||||
return ProductionMetadata("test-producer-v1", "test-restorer-v1")
|
||||
|
||||
|
||||
def _admission_process(root: Path, policy: CapacityPolicy, rank: int, control: Connection) -> None:
|
||||
try:
|
||||
store = _make_store(root, capacity=policy)
|
||||
identity = _identity(tp_rank=rank)
|
||||
original_lookup = store.lookup
|
||||
first_lookup = True
|
||||
|
||||
def notify_lookup(
|
||||
identity: WeightArtifactIdentity, *, validation: ValidationLevel, deadline: float | None = None
|
||||
) -> StoreResult:
|
||||
nonlocal first_lookup
|
||||
result = original_lookup(identity, validation=validation, deadline=deadline)
|
||||
if first_lookup:
|
||||
# Prove the caller observed a miss before the parent lets the
|
||||
# active producer publish, independent of process scheduling.
|
||||
control.send("looked_up")
|
||||
first_lookup = False
|
||||
return result
|
||||
|
||||
store.lookup = notify_lookup # type: ignore[method-assign]
|
||||
control.send("ready")
|
||||
assert control.poll(60) and control.recv() == "start"
|
||||
result = store.get_or_build(
|
||||
BuildRequest(identity),
|
||||
AdmissionProducer(identity, control),
|
||||
validation=ValidationLevel.FULL_CHECKSUM,
|
||||
deadline=time.monotonic() + 30,
|
||||
)
|
||||
if result.lease is not None:
|
||||
result.lease.close()
|
||||
control.send((result.status.value, result.failure.code.value if result.failure is not None else None))
|
||||
finally:
|
||||
control.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("admission", ["concurrent", "serialized"])
|
||||
@pytest.mark.parametrize("same_key", [False, True])
|
||||
def test_cross_identity_admission_and_capacity(
|
||||
tmp_path: Path, admission: Literal["concurrent", "serialized"], same_key: bool
|
||||
) -> None:
|
||||
policy = CapacityPolicy(max_store_bytes=192 * 1024, build_admission=admission)
|
||||
store = _make_store(tmp_path / "store", capacity=policy)
|
||||
ctx = mp.get_context("spawn")
|
||||
pairs = [ctx.Pipe() for _ in range(2)]
|
||||
processes = [
|
||||
ctx.Process(target=_admission_process, args=(store.root, policy, 0 if same_key else rank, pair[1]))
|
||||
for rank, pair in enumerate(pairs)
|
||||
]
|
||||
try:
|
||||
for process, pair in zip(processes, pairs, strict=True):
|
||||
process.start()
|
||||
pair[1].close()
|
||||
first, second = [pair[0] for pair in pairs]
|
||||
for connection in (first, second):
|
||||
assert connection.poll(60) and connection.recv() == "ready"
|
||||
first.send("start")
|
||||
assert first.poll(10) and first.recv() == "looked_up"
|
||||
assert first.poll(10) and first.recv() == "entered"
|
||||
second.send("start")
|
||||
assert second.poll(10) and second.recv() == "looked_up"
|
||||
if admission == "concurrent" and not same_key:
|
||||
assert second.poll(10) and second.recv() == "entered"
|
||||
else:
|
||||
assert not second.poll(0.2), "a second producer entered while coordination should exclude it"
|
||||
first.send("produce")
|
||||
assert first.poll(30) and first.recv() == (StoreStatus.BUILT.value, None)
|
||||
if same_key:
|
||||
assert second.poll(30) and second.recv() == (StoreStatus.JOINED.value, None)
|
||||
else:
|
||||
if admission == "serialized":
|
||||
assert second.poll(10) and second.recv() == "entered"
|
||||
second.send("produce")
|
||||
assert second.poll(30)
|
||||
assert second.recv() == (StoreStatus.FAILED.value, FailureCode.STORE_LIMIT_EXCEEDED.value)
|
||||
assert policy.max_store_bytes is not None
|
||||
assert store.inspect_domain().store_bytes < policy.max_store_bytes
|
||||
assert not list(store.tmp_dir.iterdir())
|
||||
assert {path.name for path in store.artifacts_dir.iterdir()} == {_identity(tp_rank=0).key}
|
||||
for process in processes:
|
||||
process.join(10)
|
||||
assert process.exitcode == 0
|
||||
finally:
|
||||
for process in processes:
|
||||
if process.pid is not None:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join(5)
|
||||
process.close()
|
||||
for pair in pairs:
|
||||
for connection in pair:
|
||||
connection.close()
|
||||
|
||||
|
||||
def test_admission_timeout_preserves_owner_and_warm_hit_bypasses_lock(tmp_path: Path) -> None:
|
||||
policy = CapacityPolicy(max_store_bytes=512 * 1024, build_admission="serialized")
|
||||
store = _make_store(tmp_path / "store", capacity=policy)
|
||||
identity, _ = _publish_test_artifact(store)
|
||||
other = _identity(tp_rank=1)
|
||||
lock_path = store.locks_dir / "domain-build.lock"
|
||||
with FileLock(lock_path, exclusive=True, deadline=None):
|
||||
hit = store.get_or_build(
|
||||
BuildRequest(identity),
|
||||
FakeProducer(identity),
|
||||
validation=ValidationLevel.FULL_CHECKSUM,
|
||||
deadline=time.monotonic() + 0.02,
|
||||
)
|
||||
assert hit.status is StoreStatus.HIT and hit.lease is not None
|
||||
hit.lease.close()
|
||||
timeout = store.get_or_build(
|
||||
BuildRequest(other),
|
||||
FakeProducer(other),
|
||||
validation=ValidationLevel.FULL_CHECKSUM,
|
||||
deadline=time.monotonic() + 0.02,
|
||||
)
|
||||
assert timeout.status is StoreStatus.TIMEOUT
|
||||
assert timeout.failure is not None and timeout.failure.code is FailureCode.ACTIVE_BUILD_TIMEOUT
|
||||
assert timeout.failure.retryable
|
||||
assert lock_is_active(lock_path)
|
||||
_publish_test_artifact(store, other)
|
||||
|
||||
|
||||
def test_producer_failure_releases_admission(tmp_path: Path) -> None:
|
||||
store = _make_store(
|
||||
tmp_path / "store", capacity=CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
|
||||
)
|
||||
identity = _identity()
|
||||
failed = store.get_or_build(
|
||||
BuildRequest(identity),
|
||||
FakeProducer(identity, write_mode="incomplete"),
|
||||
validation=ValidationLevel.FULL_CHECKSUM,
|
||||
deadline=time.monotonic() + 5,
|
||||
)
|
||||
assert failed.status is StoreStatus.FAILED
|
||||
assert not lock_is_active(store.locks_dir / "domain-build.lock")
|
||||
_publish_test_artifact(store, identity)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "expected"),
|
||||
[(RuntimeMode.PREFERRED, ResolutionOutcome.CANONICAL_FALLBACK), (RuntimeMode.REQUIRED, ResolutionOutcome.FAILED)],
|
||||
)
|
||||
def test_admission_timeout_obeys_runtime_mode(tmp_path: Path, mode: RuntimeMode, expected: ResolutionOutcome) -> None:
|
||||
policy = CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
|
||||
store = _make_store(tmp_path / "store", capacity=policy)
|
||||
runtime = HostWeightRuntime.from_config(
|
||||
HostWeightRuntimeConfig(mode=mode, domain=store.domain_policy, capacity=policy, wait=WaitPolicy(0.01))
|
||||
)
|
||||
identity = _identity()
|
||||
with FileLock(store.locks_dir / "domain-build.lock", exclusive=True, deadline=None):
|
||||
resolution = runtime.resolve(identity, producer=FakeProducer(identity))
|
||||
assert resolution.report.outcome is expected
|
||||
failure = resolution.report.attempts[-1].failure
|
||||
assert failure is not None and failure.code is FailureCode.ACTIVE_BUILD_TIMEOUT
|
||||
|
||||
|
||||
def test_crashed_producer_releases_admission_and_same_key_retry_recovers(tmp_path: Path) -> None:
|
||||
policy = CapacityPolicy(max_store_bytes=192 * 1024, build_admission="serialized")
|
||||
store = _make_store(tmp_path / "store", capacity=policy)
|
||||
ctx = mp.get_context("spawn")
|
||||
parent, child = ctx.Pipe()
|
||||
process = ctx.Process(target=_admission_process, args=(store.root, policy, 0, child))
|
||||
process.start()
|
||||
child.close()
|
||||
try:
|
||||
assert parent.poll(60) and parent.recv() == "ready"
|
||||
parent.send("start")
|
||||
assert parent.poll(10) and parent.recv() == "looked_up"
|
||||
assert parent.poll(10) and parent.recv() == "entered"
|
||||
parent.send("crash")
|
||||
process.join(10)
|
||||
assert process.exitcode == 27
|
||||
finally:
|
||||
if process.is_alive():
|
||||
process.kill()
|
||||
process.join(5)
|
||||
process.close()
|
||||
parent.close()
|
||||
assert not lock_is_active(store.locks_dir / "domain-build.lock")
|
||||
assert list(store.tmp_dir.iterdir())
|
||||
identity = _identity(tp_rank=0)
|
||||
recovered = store.get_or_build(
|
||||
BuildRequest(identity),
|
||||
AdmissionProducer(identity),
|
||||
validation=ValidationLevel.FULL_CHECKSUM,
|
||||
deadline=time.monotonic() + 10,
|
||||
)
|
||||
assert recovered.status is StoreStatus.BUILT and recovered.lease is not None
|
||||
assert torch.equal(recovered.lease.tensors["payload"], torch.arange(65536, dtype=torch.float32).to(torch.bfloat16))
|
||||
recovered.lease.close()
|
||||
assert not list(store.tmp_dir.iterdir())
|
||||
|
||||
|
||||
def test_admission_release_error_preserves_publication(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store = _make_store(
|
||||
tmp_path / "store", capacity=CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
|
||||
)
|
||||
original_close = FileLock.close
|
||||
|
||||
def fail_admission_close(lock: FileLock) -> None:
|
||||
original_close(lock)
|
||||
if lock.path.name == "domain-build.lock":
|
||||
raise OSError(errno.EIO, "injected admission release failure")
|
||||
|
||||
monkeypatch.setattr(FileLock, "close", fail_admission_close)
|
||||
_publish_test_artifact(store)
|
||||
assert not lock_is_active(store.locks_dir / "domain-build.lock")
|
||||
|
||||
|
||||
def test_admission_policy_is_authoritative_and_legacy_document_is_unchanged(tmp_path: Path) -> None:
|
||||
concurrent = CapacityPolicy(max_store_bytes=16384)
|
||||
serialized = CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
|
||||
legacy = _make_store(tmp_path / "legacy", capacity=concurrent)
|
||||
path = legacy.root / "domain-policy.json"
|
||||
original = path.read_bytes()
|
||||
assert json.loads(original) == {
|
||||
"schema_version": 1,
|
||||
"policy_version": 1,
|
||||
"max_artifact_bytes": None,
|
||||
"max_store_bytes": 16384,
|
||||
"min_free_bytes": 0,
|
||||
"eviction": "none",
|
||||
}
|
||||
with pytest.raises(HostWeightError, match="incompatible schema"):
|
||||
_make_store(legacy.root, capacity=serialized)
|
||||
assert path.read_bytes() == original
|
||||
strict = _make_store(tmp_path / "serialized", capacity=serialized)
|
||||
document = json.loads((strict.root / "domain-policy.json").read_bytes())
|
||||
assert document["schema_version"] == 2 and document["build_admission"] == "serialized"
|
||||
assert _make_store(strict.root, capacity=serialized).domain_uuid == strict.domain_uuid
|
||||
with pytest.raises(HostWeightError, match="incompatible schema"):
|
||||
_make_store(strict.root, capacity=concurrent)
|
||||
|
||||
|
||||
def test_admission_policy_validation() -> None:
|
||||
with pytest.raises(ValueError, match="requires max_store_bytes"):
|
||||
CapacityPolicy(build_admission="serialized")
|
||||
with pytest.raises(ValueError, match="must be concurrent or serialized"):
|
||||
CapacityPolicy(build_admission="unknown") # type: ignore[arg-type]
|
||||
@@ -8,7 +8,6 @@ import math
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
|
||||
class RuntimeMode(str, Enum):
|
||||
@@ -67,7 +66,6 @@ class CapacityPolicy:
|
||||
max_store_bytes: int | None = None
|
||||
min_free_bytes: int = 0
|
||||
eviction: str = "none"
|
||||
build_admission: Literal["concurrent", "serialized"] = "concurrent"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name, value in (
|
||||
@@ -80,10 +78,6 @@ class CapacityPolicy:
|
||||
raise ValueError("min_free_bytes must not be negative")
|
||||
if not isinstance(self.eviction, str) or self.eviction != "none":
|
||||
raise ValueError("automatic host weight eviction is not implemented")
|
||||
if self.build_admission not in ("concurrent", "serialized"):
|
||||
raise ValueError("build_admission must be concurrent or serialized")
|
||||
if self.build_admission == "serialized" and self.max_store_bytes is None:
|
||||
raise ValueError("serialized build admission requires max_store_bytes")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -627,7 +627,7 @@ class FilesystemHostWeightStore:
|
||||
self.policy_version = 1
|
||||
|
||||
def _capacity_document(self, *, policy_version: int) -> dict[str, object]:
|
||||
document: dict[str, object] = {
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"policy_version": policy_version,
|
||||
"max_artifact_bytes": self.capacity_policy.max_artifact_bytes,
|
||||
@@ -635,9 +635,6 @@ class FilesystemHostWeightStore:
|
||||
"min_free_bytes": self.capacity_policy.min_free_bytes,
|
||||
"eviction": self.capacity_policy.eviction,
|
||||
}
|
||||
if self.capacity_policy.build_admission == "serialized":
|
||||
document.update(schema_version=2, build_admission="serialized")
|
||||
return document
|
||||
|
||||
def _build_lock_path(self, key: str) -> Path:
|
||||
return self.locks_dir / f"{key}.build.lock"
|
||||
@@ -1016,40 +1013,6 @@ class FilesystemHostWeightStore:
|
||||
if initial.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}:
|
||||
return initial
|
||||
|
||||
if self.capacity_policy.build_admission == "concurrent":
|
||||
return self._build_after_miss(identity, producer, validation=validation, deadline=deadline)
|
||||
try:
|
||||
admission = FileLock(self.locks_dir / "domain-build.lock", exclusive=True, deadline=deadline)
|
||||
except FileLockTimeoutError as exc:
|
||||
return StoreResult(
|
||||
StoreStatus.TIMEOUT,
|
||||
failure=_failure(
|
||||
ResolutionStage.PRODUCTION, FailureCode.ACTIVE_BUILD_TIMEOUT, str(exc), retryable=True
|
||||
),
|
||||
)
|
||||
except OSError as exc:
|
||||
return StoreResult(
|
||||
StoreStatus.FAILED,
|
||||
failure=_failure(ResolutionStage.DOMAIN, FailureCode.DOMAIN_UNAVAILABLE, str(exc), retryable=True),
|
||||
)
|
||||
try:
|
||||
return self._build_after_miss(identity, producer, validation=validation, deadline=deadline)
|
||||
finally:
|
||||
try:
|
||||
admission.close()
|
||||
except OSError:
|
||||
# Releasing coordination must not revise the completed build's
|
||||
# publication outcome or replace its primary failure.
|
||||
logger.warning("Failed to release domain build admission under %s", self.root, exc_info=True)
|
||||
|
||||
def _build_after_miss(
|
||||
self,
|
||||
identity: WeightArtifactIdentity,
|
||||
producer: WeightProducer,
|
||||
*,
|
||||
validation: ValidationLevel,
|
||||
deadline: float,
|
||||
) -> StoreResult:
|
||||
try:
|
||||
build_lock = FileLock(self._build_lock_path(identity.key), exclusive=True, deadline=deadline)
|
||||
except FileLockTimeoutError as exc:
|
||||
|
||||
Reference in new issue
Block a user