Compare commits

..
Author SHA1 Message Date
Hongsheng Liu 380127049a [Doc] Fix HWR diagnostic link in strict docs builds
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
2026-09-07 21:39:37 +08:00
Hongsheng Liu 743953b821 [Misc] Make retention probe tensor size configurable
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
2026-09-06 21:53:50 +08:00
Hongsheng Liu 86166fd988 [Misc] Add standalone safetensors retention diagnostic
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
2026-09-06 16:57:46 +08:00
7 changed files with 186 additions and 394 deletions

No files matched your search

+57
View File
@@ -0,0 +1,57 @@
# Host weight dependency memory diagnostic
`safetensors_retention.py` isolates repeated CPU `get_tensor()` calls from
access to one cached tensor view. It imports neither vLLM nor HWR and uses a
synthetic float32 payload controlled by `--tensor-elements` (default: 64).
Tensor size can affect retention: a negative result for 16 elements does not
rule out growth for 64 elements. The selected size is recorded in the JSON
arguments and used for both file creation and the correctness check.
HWR itself calls `get_tensor()` when
acquiring a lease, then exposes cached views; this probe does not measure HWR
requests or repeated lease acquisition.
Run one mode per fresh process in the same Python environment and CPU affinity:
```bash
git rev-parse HEAD > revision.txt
git status --short > working-tree.txt
git diff > working-tree.patch
# Select an allowed CPU from your process affinity; 0 is only an example.
CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
benchmarks/host_weight_runtime/safetensors_retention.py \
--mode get_tensor --iterations 10000 --sample-every 5000 > feasibility.json
```
For a size comparison, run 16 and 64 elements twice each in fresh processes,
with cached reuse as a control at both sizes:
```bash
for repetition in 1 2; do
for elements in 16 64; do
for mode in get_tensor reuse; do
CUDA_VISIBLE_DEVICES='' taskset -c 0 timeout 120s python \
benchmarks/host_weight_runtime/safetensors_retention.py \
--mode "$mode" --tensor-elements "$elements" \
> "$mode-$elements-$repetition.json"
done
done
done
```
The diagnostic sets Torch to one CPU thread, warms up before the baseline,
and reports preparation separately from timed loop work. Checkpoints run GC
and report Linux private/anonymous memory, RSS, and open descriptors. It checks
that transient tensor objects and the final cached view are released. Linux
`/proc/self/smaps_rollup` must be readable; invalid arguments and missing proc
support fail explicitly. Keep raw JSON and the repository snapshot with results.
Compare the final loop sample with the warmed baseline, then inspect the closed
sample. Private-memory growth after Python tensor objects disappear is evidence
of retention in the dependency/allocator path, not proof of a leak's root cause.
RSS can include file-backed pages; it is not a private-memory metric. Samples
also include small diagnostic bookkeeping allocations. Do not assert a portable
memory threshold or turn this synthetic loop into a per-request service estimate.
Do not drop shared caches, trim allocators, or add serving-time GC to make the
numbers look smaller. A dependency replacement or version change requires its
own reproducible comparison and correctness/lifecycle validation.
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Linux CPU diagnostic: repeated safetensors materialization versus view reuse."""
from __future__ import annotations
import argparse
import gc
import json
import os
import platform
import tempfile
import time
import weakref
from pathlib import Path
def positive_int(value: str) -> int:
parsed = int(value)
if parsed <= 0:
raise argparse.ArgumentTypeError("must be positive")
return parsed
def sample(phase: str, iterations: int, loop_seconds: float) -> dict[str, object]:
gc.collect()
counters = {}
for line in Path("/proc/self/smaps_rollup").read_text().splitlines():
fields = line.split()
if len(fields) == 3 and fields[2] == "kB":
counters[fields[0].removesuffix(":")] = int(fields[1]) * 1024
return {
"phase": phase,
"iterations": iterations,
"loop_seconds": loop_seconds,
"rss_bytes": counters["Rss"],
"private_bytes": counters["Private_Clean"] + counters["Private_Dirty"],
"anonymous_bytes": counters["Anonymous"],
"fd_count": len(tuple(Path("/proc/self/fd").iterdir())),
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--mode", choices=("get_tensor", "reuse"), required=True)
parser.add_argument(
"--tensor-elements", type=positive_int, default=64, help="Number of float32 elements in the synthetic tensor"
)
parser.add_argument("--iterations", type=positive_int, default=1_000_000)
parser.add_argument("--sample-every", type=positive_int, default=100_000)
parser.add_argument("--warmup", type=positive_int, default=1000)
args = parser.parse_args()
if not Path("/proc/self/smaps_rollup").is_file():
parser.error("requires Linux /proc/self/smaps_rollup")
preparation_start = time.monotonic()
import safetensors
import torch
from safetensors import safe_open
from safetensors.torch import save_file
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
samples = [sample("before_file", 0, 0.0)]
with tempfile.TemporaryDirectory(prefix="safetensors-retention-") as directory:
path = Path(directory) / "synthetic.safetensors"
save_file({"weight": torch.arange(args.tensor_elements, dtype=torch.float32)}, str(path))
with safe_open(path, framework="pt", device="cpu") as reader:
cached = reader.get_tensor("weight")
assert torch.equal(cached, torch.arange(args.tensor_elements, dtype=torch.float32))
for _ in range(args.warmup):
tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
del tensor
samples.append(sample("baseline", 0, 0.0))
preparation_seconds = time.monotonic() - preparation_start
completed = 0
loop_seconds = 0.0
while completed < args.iterations:
count = min(args.sample_every, args.iterations - completed)
start = time.monotonic()
for _ in range(count):
tensor = reader.get_tensor("weight") if args.mode == "get_tensor" else cached
last_tensor = weakref.ref(tensor)
del tensor
loop_seconds += time.monotonic() - start
completed += count
snapshot = sample("loop", completed, loop_seconds)
snapshot["last_tensor_alive"] = last_tensor() is not None
if args.mode == "get_tensor":
assert last_tensor() is None, "diagnostic retained a transient tensor"
samples.append(snapshot)
del cached
del reader
samples.append(sample("closed", completed, loop_seconds))
assert last_tensor() is None, "last tensor survived reader/view teardown"
print(
json.dumps(
{
"schema_version": 1,
"arguments": vars(args),
"python": platform.python_version(),
"torch": torch.__version__,
"safetensors": safetensors.__version__,
"kernel": platform.release(),
"hostname": platform.node(),
"pid": os.getpid(),
"cpu_affinity": sorted(os.sched_getaffinity(0)),
"torch_threads": torch.get_num_threads(),
"pythonmalloc": os.environ.get("PYTHONMALLOC", "default"),
"preparation_seconds": preparation_seconds,
"samples": samples,
},
indent=2,
)
)
if __name__ == "__main__":
main()
+5 -10
View File
@@ -42,6 +42,11 @@ V1 does not include:
## Motivation and use cases
For investigating CPU memory retained by dependency tensor materialization,
see the [standalone safetensors diagnostic](https://github.com/vllm-project/vllm-omni/blob/main/benchmarks/host_weight_runtime/README.md).
It distinguishes repeated dependency calls from reuse of cached views and does
not establish per-request HWR leakage.
Model loading can create the same final host representation repeatedly. This
is especially expensive when loading performs checkpoint decoding, tensor
renaming, TP slicing, quantization, packing, or scale construction before GPU
@@ -313,16 +318,6 @@ One process per exact identity owns a build; other workers wait and then acquire
leases for the published artifact. Publication is invisible until all payloads
and metadata are validated, hashed, fsynced, and atomically renamed.
By default, different identities may build concurrently. A domain configured
with `CapacityPolicy(build_admission="serialized", max_store_bytes=...)` admits
only one producer/publication operation at a time. Warm hits remain concurrent,
and admission uses the existing coordination deadline. All workers must agree
on the serialized domain's schema-2 capacity policy; use a new root when opting
in, since in-place migration of an active domain is unsupported. This prevents
competing producers from racing allocation checks but does not provide a
filesystem-wide byte quota or automatic stale-data reclamation. See the
[module capacity contract](../module/host_weight_runtime.md#validation-and-capacity).
`coordination_timeout_seconds` bounds filesystem lock acquisition. It does not
cancel synchronous validation, a producer that has already started, or atomic
publication. A hung in-process producer therefore blocks its owning process and
+3 -35
View File
@@ -311,41 +311,9 @@ point-in-time evidence only; kernel locks are not a persistent owner registry.
Capacity policy covers all store-owned bytes, including ready, temporary, and
quarantined data. The local writer preflights `max_artifact_bytes`,
`max_store_bytes`, and `min_free_bytes`, and preallocates payloads where the
filesystem supports it. `ENOSPC` remains a normal typed store failure.
`CapacityPolicy.build_admission` defaults to `concurrent`, preserving parallel
production for different identities and best-effort capacity preflight. An
explicit `serialized` policy, which requires `max_store_bytes`, admits one
cooperative build/publication operation per domain through
`locks/domain-build.lock`. Production lock order becomes domain admission,
per-key build, then artifact. Validated warm hits bypass admission; queued
same-key callers still recheck and join a completed artifact. Admission waiting
uses the caller's existing coordination deadline and returns a retryable build
timeout without interrupting its owner.
The kernel releases admission on process exit. An explicit same-key retry uses
the existing stale-temp recovery; files from unrelated dead builds continue to
count toward capacity. A synchronous producer that hangs still requires
external process supervision. Admission release errors are logged without
revising an already determined publication result.
Concurrent domains retain their exact schema-1 capacity document. Serialized
domains use a schema-2 capacity document with `build_admission=serialized`, so
mixed policies and old readers fail initialization rather than bypass admission.
Opt into serialization using a new root with an agreed policy; in-place policy
migration while old workers may exist is unsupported. For example:
```python
from vllm_omni.host_weight_runtime import CapacityPolicy
capacity = CapacityPolicy(max_store_bytes=192 * 1024**3, build_admission="serialized")
```
Serialization prevents cooperative producers from racing each other's
allocation checks. It is not a filesystem quota: control metadata, finalization,
and outside writers can still affect total usage. Automatic eviction and
parallel byte reservations are not implemented. This policy trades parallel
cold-build throughput for predictable producer admission.
filesystem supports it. `ENOSPC` remains a normal typed store failure because
concurrent preflight is inherently racy. Automatic eviction and strict
concurrent reservations are not implemented.
For tmpfs, artifact bytes are host-memory consumption and may consume swap;
they must not be reported operationally as ordinary disk capacity. The store
@@ -1,305 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU process and policy coverage for domain-wide producer admission."""
from __future__ import annotations
import errno
import json
import multiprocessing as mp
import os
import time
from multiprocessing.connection import Connection
from pathlib import Path
from typing import Literal
import pytest
import torch
from tests.host_weight_runtime.test_filesystem_store import FakeProducer, _identity, _make_store, _publish_test_artifact
from vllm_omni.host_weight_runtime import (
BuildRequest,
CapacityPolicy,
FailureCode,
HostWeightError,
HostWeightRuntime,
HostWeightRuntimeConfig,
ProductionMetadata,
ResolutionOutcome,
RuntimeMode,
StoreResult,
StoreStatus,
TensorWriteSpec,
ValidationLevel,
WaitPolicy,
WeightArtifactIdentity,
WeightProductionSpec,
)
from vllm_omni.host_weight_runtime.filesystem.locks import FileLock, lock_is_active
from vllm_omni.host_weight_runtime.protocols import ArtifactWriter
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class AdmissionProducer:
def __init__(self, identity: WeightArtifactIdentity, control: Connection | None = None) -> None:
self._spec = FakeProducer(identity).spec
self.control = control
@property
def spec(self) -> WeightProductionSpec:
return self._spec
def produce(self, writer: ArtifactWriter) -> ProductionMetadata:
action = "produce"
if self.control is not None:
self.control.send("entered")
if not self.control.poll(60):
raise TimeoutError("test did not release producer")
action = self.control.recv()
spec = TensorWriteSpec("payload", (65536,), torch.bfloat16)
with writer.open_tensor_file("weights.safetensors", (spec,)) as output:
output.write_tensor("payload", torch.arange(65536, dtype=torch.float32).to(torch.bfloat16))
if action == "crash":
os._exit(27)
return ProductionMetadata("test-producer-v1", "test-restorer-v1")
def _admission_process(root: Path, policy: CapacityPolicy, rank: int, control: Connection) -> None:
try:
store = _make_store(root, capacity=policy)
identity = _identity(tp_rank=rank)
original_lookup = store.lookup
first_lookup = True
def notify_lookup(
identity: WeightArtifactIdentity, *, validation: ValidationLevel, deadline: float | None = None
) -> StoreResult:
nonlocal first_lookup
result = original_lookup(identity, validation=validation, deadline=deadline)
if first_lookup:
# Prove the caller observed a miss before the parent lets the
# active producer publish, independent of process scheduling.
control.send("looked_up")
first_lookup = False
return result
store.lookup = notify_lookup # type: ignore[method-assign]
control.send("ready")
assert control.poll(60) and control.recv() == "start"
result = store.get_or_build(
BuildRequest(identity),
AdmissionProducer(identity, control),
validation=ValidationLevel.FULL_CHECKSUM,
deadline=time.monotonic() + 30,
)
if result.lease is not None:
result.lease.close()
control.send((result.status.value, result.failure.code.value if result.failure is not None else None))
finally:
control.close()
@pytest.mark.parametrize("admission", ["concurrent", "serialized"])
@pytest.mark.parametrize("same_key", [False, True])
def test_cross_identity_admission_and_capacity(
tmp_path: Path, admission: Literal["concurrent", "serialized"], same_key: bool
) -> None:
policy = CapacityPolicy(max_store_bytes=192 * 1024, build_admission=admission)
store = _make_store(tmp_path / "store", capacity=policy)
ctx = mp.get_context("spawn")
pairs = [ctx.Pipe() for _ in range(2)]
processes = [
ctx.Process(target=_admission_process, args=(store.root, policy, 0 if same_key else rank, pair[1]))
for rank, pair in enumerate(pairs)
]
try:
for process, pair in zip(processes, pairs, strict=True):
process.start()
pair[1].close()
first, second = [pair[0] for pair in pairs]
for connection in (first, second):
assert connection.poll(60) and connection.recv() == "ready"
first.send("start")
assert first.poll(10) and first.recv() == "looked_up"
assert first.poll(10) and first.recv() == "entered"
second.send("start")
assert second.poll(10) and second.recv() == "looked_up"
if admission == "concurrent" and not same_key:
assert second.poll(10) and second.recv() == "entered"
else:
assert not second.poll(0.2), "a second producer entered while coordination should exclude it"
first.send("produce")
assert first.poll(30) and first.recv() == (StoreStatus.BUILT.value, None)
if same_key:
assert second.poll(30) and second.recv() == (StoreStatus.JOINED.value, None)
else:
if admission == "serialized":
assert second.poll(10) and second.recv() == "entered"
second.send("produce")
assert second.poll(30)
assert second.recv() == (StoreStatus.FAILED.value, FailureCode.STORE_LIMIT_EXCEEDED.value)
assert policy.max_store_bytes is not None
assert store.inspect_domain().store_bytes < policy.max_store_bytes
assert not list(store.tmp_dir.iterdir())
assert {path.name for path in store.artifacts_dir.iterdir()} == {_identity(tp_rank=0).key}
for process in processes:
process.join(10)
assert process.exitcode == 0
finally:
for process in processes:
if process.pid is not None:
if process.is_alive():
process.kill()
process.join(5)
process.close()
for pair in pairs:
for connection in pair:
connection.close()
def test_admission_timeout_preserves_owner_and_warm_hit_bypasses_lock(tmp_path: Path) -> None:
policy = CapacityPolicy(max_store_bytes=512 * 1024, build_admission="serialized")
store = _make_store(tmp_path / "store", capacity=policy)
identity, _ = _publish_test_artifact(store)
other = _identity(tp_rank=1)
lock_path = store.locks_dir / "domain-build.lock"
with FileLock(lock_path, exclusive=True, deadline=None):
hit = store.get_or_build(
BuildRequest(identity),
FakeProducer(identity),
validation=ValidationLevel.FULL_CHECKSUM,
deadline=time.monotonic() + 0.02,
)
assert hit.status is StoreStatus.HIT and hit.lease is not None
hit.lease.close()
timeout = store.get_or_build(
BuildRequest(other),
FakeProducer(other),
validation=ValidationLevel.FULL_CHECKSUM,
deadline=time.monotonic() + 0.02,
)
assert timeout.status is StoreStatus.TIMEOUT
assert timeout.failure is not None and timeout.failure.code is FailureCode.ACTIVE_BUILD_TIMEOUT
assert timeout.failure.retryable
assert lock_is_active(lock_path)
_publish_test_artifact(store, other)
def test_producer_failure_releases_admission(tmp_path: Path) -> None:
store = _make_store(
tmp_path / "store", capacity=CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
)
identity = _identity()
failed = store.get_or_build(
BuildRequest(identity),
FakeProducer(identity, write_mode="incomplete"),
validation=ValidationLevel.FULL_CHECKSUM,
deadline=time.monotonic() + 5,
)
assert failed.status is StoreStatus.FAILED
assert not lock_is_active(store.locks_dir / "domain-build.lock")
_publish_test_artifact(store, identity)
@pytest.mark.parametrize(
("mode", "expected"),
[(RuntimeMode.PREFERRED, ResolutionOutcome.CANONICAL_FALLBACK), (RuntimeMode.REQUIRED, ResolutionOutcome.FAILED)],
)
def test_admission_timeout_obeys_runtime_mode(tmp_path: Path, mode: RuntimeMode, expected: ResolutionOutcome) -> None:
policy = CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
store = _make_store(tmp_path / "store", capacity=policy)
runtime = HostWeightRuntime.from_config(
HostWeightRuntimeConfig(mode=mode, domain=store.domain_policy, capacity=policy, wait=WaitPolicy(0.01))
)
identity = _identity()
with FileLock(store.locks_dir / "domain-build.lock", exclusive=True, deadline=None):
resolution = runtime.resolve(identity, producer=FakeProducer(identity))
assert resolution.report.outcome is expected
failure = resolution.report.attempts[-1].failure
assert failure is not None and failure.code is FailureCode.ACTIVE_BUILD_TIMEOUT
def test_crashed_producer_releases_admission_and_same_key_retry_recovers(tmp_path: Path) -> None:
policy = CapacityPolicy(max_store_bytes=192 * 1024, build_admission="serialized")
store = _make_store(tmp_path / "store", capacity=policy)
ctx = mp.get_context("spawn")
parent, child = ctx.Pipe()
process = ctx.Process(target=_admission_process, args=(store.root, policy, 0, child))
process.start()
child.close()
try:
assert parent.poll(60) and parent.recv() == "ready"
parent.send("start")
assert parent.poll(10) and parent.recv() == "looked_up"
assert parent.poll(10) and parent.recv() == "entered"
parent.send("crash")
process.join(10)
assert process.exitcode == 27
finally:
if process.is_alive():
process.kill()
process.join(5)
process.close()
parent.close()
assert not lock_is_active(store.locks_dir / "domain-build.lock")
assert list(store.tmp_dir.iterdir())
identity = _identity(tp_rank=0)
recovered = store.get_or_build(
BuildRequest(identity),
AdmissionProducer(identity),
validation=ValidationLevel.FULL_CHECKSUM,
deadline=time.monotonic() + 10,
)
assert recovered.status is StoreStatus.BUILT and recovered.lease is not None
assert torch.equal(recovered.lease.tensors["payload"], torch.arange(65536, dtype=torch.float32).to(torch.bfloat16))
recovered.lease.close()
assert not list(store.tmp_dir.iterdir())
def test_admission_release_error_preserves_publication(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
store = _make_store(
tmp_path / "store", capacity=CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
)
original_close = FileLock.close
def fail_admission_close(lock: FileLock) -> None:
original_close(lock)
if lock.path.name == "domain-build.lock":
raise OSError(errno.EIO, "injected admission release failure")
monkeypatch.setattr(FileLock, "close", fail_admission_close)
_publish_test_artifact(store)
assert not lock_is_active(store.locks_dir / "domain-build.lock")
def test_admission_policy_is_authoritative_and_legacy_document_is_unchanged(tmp_path: Path) -> None:
concurrent = CapacityPolicy(max_store_bytes=16384)
serialized = CapacityPolicy(max_store_bytes=16384, build_admission="serialized")
legacy = _make_store(tmp_path / "legacy", capacity=concurrent)
path = legacy.root / "domain-policy.json"
original = path.read_bytes()
assert json.loads(original) == {
"schema_version": 1,
"policy_version": 1,
"max_artifact_bytes": None,
"max_store_bytes": 16384,
"min_free_bytes": 0,
"eviction": "none",
}
with pytest.raises(HostWeightError, match="incompatible schema"):
_make_store(legacy.root, capacity=serialized)
assert path.read_bytes() == original
strict = _make_store(tmp_path / "serialized", capacity=serialized)
document = json.loads((strict.root / "domain-policy.json").read_bytes())
assert document["schema_version"] == 2 and document["build_admission"] == "serialized"
assert _make_store(strict.root, capacity=serialized).domain_uuid == strict.domain_uuid
with pytest.raises(HostWeightError, match="incompatible schema"):
_make_store(strict.root, capacity=concurrent)
def test_admission_policy_validation() -> None:
with pytest.raises(ValueError, match="requires max_store_bytes"):
CapacityPolicy(build_admission="serialized")
with pytest.raises(ValueError, match="must be concurrent or serialized"):
CapacityPolicy(build_admission="unknown") # type: ignore[arg-type]
-6
View File
@@ -8,7 +8,6 @@ import math
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Literal
class RuntimeMode(str, Enum):
@@ -67,7 +66,6 @@ class CapacityPolicy:
max_store_bytes: int | None = None
min_free_bytes: int = 0
eviction: str = "none"
build_admission: Literal["concurrent", "serialized"] = "concurrent"
def __post_init__(self) -> None:
for name, value in (
@@ -80,10 +78,6 @@ class CapacityPolicy:
raise ValueError("min_free_bytes must not be negative")
if not isinstance(self.eviction, str) or self.eviction != "none":
raise ValueError("automatic host weight eviction is not implemented")
if self.build_admission not in ("concurrent", "serialized"):
raise ValueError("build_admission must be concurrent or serialized")
if self.build_admission == "serialized" and self.max_store_bytes is None:
raise ValueError("serialized build admission requires max_store_bytes")
@dataclass(frozen=True)
@@ -627,7 +627,7 @@ class FilesystemHostWeightStore:
self.policy_version = 1
def _capacity_document(self, *, policy_version: int) -> dict[str, object]:
document: dict[str, object] = {
return {
"schema_version": 1,
"policy_version": policy_version,
"max_artifact_bytes": self.capacity_policy.max_artifact_bytes,
@@ -635,9 +635,6 @@ class FilesystemHostWeightStore:
"min_free_bytes": self.capacity_policy.min_free_bytes,
"eviction": self.capacity_policy.eviction,
}
if self.capacity_policy.build_admission == "serialized":
document.update(schema_version=2, build_admission="serialized")
return document
def _build_lock_path(self, key: str) -> Path:
return self.locks_dir / f"{key}.build.lock"
@@ -1016,40 +1013,6 @@ class FilesystemHostWeightStore:
if initial.status in {StoreStatus.TIMEOUT, StoreStatus.FAILED}:
return initial
if self.capacity_policy.build_admission == "concurrent":
return self._build_after_miss(identity, producer, validation=validation, deadline=deadline)
try:
admission = FileLock(self.locks_dir / "domain-build.lock", exclusive=True, deadline=deadline)
except FileLockTimeoutError as exc:
return StoreResult(
StoreStatus.TIMEOUT,
failure=_failure(
ResolutionStage.PRODUCTION, FailureCode.ACTIVE_BUILD_TIMEOUT, str(exc), retryable=True
),
)
except OSError as exc:
return StoreResult(
StoreStatus.FAILED,
failure=_failure(ResolutionStage.DOMAIN, FailureCode.DOMAIN_UNAVAILABLE, str(exc), retryable=True),
)
try:
return self._build_after_miss(identity, producer, validation=validation, deadline=deadline)
finally:
try:
admission.close()
except OSError:
# Releasing coordination must not revise the completed build's
# publication outcome or replace its primary failure.
logger.warning("Failed to release domain build admission under %s", self.root, exc_info=True)
def _build_after_miss(
self,
identity: WeightArtifactIdentity,
producer: WeightProducer,
*,
validation: ValidationLevel,
deadline: float,
) -> StoreResult:
try:
build_lock = FileLock(self._build_lock_path(identity.key), exclusive=True, deadline=deadline)
except FileLockTimeoutError as exc: