[Docs] Guide for using sleep mode and enable sleep mode (#660)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
knlnguyen1802andJiangyun Zhu authored and GitHub committed 2026-01-06 17:31:51 +08:00
1 parent b7e63e07d7
commit c32235c4e0
3 files changed
+62

No files matched your search

+21
View File
@@ -143,6 +143,16 @@ class GPUWorker:
return self.pipeline.load_weights(weights)
def sleep(self, level: int = 1) -> bool:
"""
Put the worker to sleep. The worker should not process any requests.
The caller should guarantee that no requests are being processed
during the sleep period, before `wake_up` is called.
Args:
level: The sleep level. Level 1 sleep will offload the model
weights and discard the kv cache.
Currently only support level 1.
"""
from vllm.device_allocator.cumem import CuMemAllocator
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
@@ -166,6 +176,17 @@ class GPUWorker:
return True
def wake_up(self, tags: list[str] | None = None) -> bool:
"""
Wake up the worker from sleep mode. See the sleep function
method for more details.
Args:
tags: An optional list of tags to reallocate the worker memory
for specific memory allocations. Values must be in
`("weights")`. If None, all memory is reallocated.
wake_up should be called with all tags (or None) before the
worker is used again.
"""
from vllm.device_allocator.cumem import CuMemAllocator
allocator = CuMemAllocator.get_instance()