[Model] Derive the retention budget from observed boundary peaks

The staged-bytes seed left the transient half of the working set to a
guessed multiplier, and that half scales with the running workload rather
than the model topology. Track the largest cached-but-unallocated value
observed at any component boundary instead: the budget is the larger of
the seed and the observed peak times a 1.25 margin, still capped at a
quarter of device capacity. A workload that outgrows the seed pays one
release while the observation lands, then retains steadily. No warmup
flag is needed and no user-chosen fraction exists.

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-08-26 12:52:19 +08:00
1 parent c5295f3ffe
commit 27f9726a6a
3 files changed
+84 -32

No files matched your search

@@ -295,3 +295,28 @@ def test_allocator_cache_releases_when_telemetry_is_unavailable(monkeypatch, pat
assert retention.release_if_needed()
empty_cache.assert_called_once_with()
def test_allocator_cache_budget_adapts_to_observed_peak(monkeypatch, patched_runtime):
empty_cache, _ = patched_runtime
telemetry = {"reserved": 40, "allocated": 10, "free": 80}
monkeypatch.setattr(torch.accelerator, "memory_reserved", lambda _device: telemetry["reserved"])
monkeypatch.setattr(torch.accelerator, "memory_allocated", lambda _device: telemetry["allocated"])
monkeypatch.setattr(
residency_module.current_omni_platform,
"get_device_memory",
lambda _device: (telemetry["free"], 100),
)
retention = BoundedAllocatorCache(torch.device("cpu"))
retention.grow_retention_budget(10)
# First boundary: cached 30 exceeds the seed budget 20, so the policy
# releases once and records the observed peak.
assert retention.release_if_needed()
assert retention.observed_cached_peak == 30
# The observation raises the budget to 30 * 1.25 = 37, so the same cached
# level is retained on every later boundary without another release.
assert not retention.release_if_needed()
assert not retention.release_if_needed()
assert empty_cache.call_count == 1