[Model] Derive MiniMax-H3 DLO component cache budget from staged bytes
Replace BoundedAllocatorCache's device-capacity fraction bound (25% of total memory) with a budget derived from the model topology: each attached PinnedModuleStager reports its unique staged storage bytes once, and the retention ceiling is that total times a fixed 2.0 headroom multiplier covering allocator fragmentation and transient storages freed at component boundaries. The 5% physically-free floor is unchanged. The budget therefore scales with the staged components instead of device capacity, so no user-chosen fraction is needed and behavior is identical across differently sized devices with the same model. Re-attaching the same cache does not double count; a second stager grows the budget by its own bytes. Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
780e8d40aa
commit
ba5943dddb
3 files changed
+72
-12
No files matched your search
@@ -210,6 +210,7 @@ def test_allocator_cache_policy_enforces_cache_and_free_memory_bounds(
|
||||
lambda _device: (free, 100),
|
||||
)
|
||||
retention = BoundedAllocatorCache(torch.device("cpu"))
|
||||
retention.grow_retention_budget(10)
|
||||
|
||||
released = retention.release_if_needed()
|
||||
|
||||
@@ -217,6 +218,31 @@ def test_allocator_cache_policy_enforces_cache_and_free_memory_bounds(
|
||||
assert empty_cache.call_count == int(expected_release)
|
||||
|
||||
|
||||
def test_allocator_cache_budget_is_derived_from_attached_stager_bytes(patched_runtime):
|
||||
left, right = _aliased_modules()
|
||||
retention = BoundedAllocatorCache(torch.device("cpu"))
|
||||
assert retention.budget_bytes == 0
|
||||
|
||||
stager = PinnedModuleStager(
|
||||
[left, right],
|
||||
torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
cache_retention=retention,
|
||||
)
|
||||
expected_bytes = 12 * 4 + 8 * 8 # float32 parameter + int64 buffer storages
|
||||
assert stager.staged_bytes == expected_bytes
|
||||
assert retention.budget_bytes == 2 * expected_bytes
|
||||
|
||||
# Re-attaching the same cache does not double count.
|
||||
stager.set_cache_retention(retention)
|
||||
assert retention.budget_bytes == 2 * expected_bytes
|
||||
|
||||
# A second stager grows the budget by its own staged bytes.
|
||||
extra = PinnedModuleStager(torch.nn.Linear(2, 2), torch.device("cpu"), pin_memory=False)
|
||||
extra.set_cache_retention(retention)
|
||||
assert retention.budget_bytes == 2 * (expected_bytes + extra.staged_bytes)
|
||||
|
||||
|
||||
def test_allocator_cache_releases_when_telemetry_is_unavailable(monkeypatch, patched_runtime):
|
||||
empty_cache, _ = patched_runtime
|
||||
|
||||
|
||||
Reference in new issue
Block a user