[Model] Derive MiniMax-H3 DLO component cache budget from staged bytes

Replace BoundedAllocatorCache's device-capacity fraction bound (25% of
total memory) with a budget derived from the model topology: each attached
PinnedModuleStager reports its unique staged storage bytes once, and the
retention ceiling is that total times a fixed 2.0 headroom multiplier
covering allocator fragmentation and transient storages freed at component
boundaries. The 5% physically-free floor is unchanged.

The budget therefore scales with the staged components instead of device
capacity, so no user-chosen fraction is needed and behavior is identical
across differently sized devices with the same model. Re-attaching the
same cache does not double count; a second stager grows the budget by its
own bytes.

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-08-26 09:49:55 +08:00
1 parent 780e8d40aa
commit ba5943dddb
3 files changed
+72 -12

No files matched your search

@@ -210,6 +210,7 @@ def test_allocator_cache_policy_enforces_cache_and_free_memory_bounds(
lambda _device: (free, 100),
)
retention = BoundedAllocatorCache(torch.device("cpu"))
retention.grow_retention_budget(10)
released = retention.release_if_needed()
@@ -217,6 +218,31 @@ def test_allocator_cache_policy_enforces_cache_and_free_memory_bounds(
assert empty_cache.call_count == int(expected_release)
def test_allocator_cache_budget_is_derived_from_attached_stager_bytes(patched_runtime):
left, right = _aliased_modules()
retention = BoundedAllocatorCache(torch.device("cpu"))
assert retention.budget_bytes == 0
stager = PinnedModuleStager(
[left, right],
torch.device("cpu"),
pin_memory=False,
cache_retention=retention,
)
expected_bytes = 12 * 4 + 8 * 8 # float32 parameter + int64 buffer storages
assert stager.staged_bytes == expected_bytes
assert retention.budget_bytes == 2 * expected_bytes
# Re-attaching the same cache does not double count.
stager.set_cache_retention(retention)
assert retention.budget_bytes == 2 * expected_bytes
# A second stager grows the budget by its own staged bytes.
extra = PinnedModuleStager(torch.nn.Linear(2, 2), torch.device("cpu"), pin_memory=False)
extra.set_cache_retention(retention)
assert retention.budget_bytes == 2 * (expected_bytes + extra.staged_bytes)
def test_allocator_cache_releases_when_telemetry_is_unavailable(monkeypatch, patched_runtime):
empty_cache, _ = patched_runtime