[Docs] Guide for using sleep mode and enable sleep mode (#660)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
b7e63e07d7
commit
c32235c4e0
3 files changed
+62
No files matched your search
@@ -143,6 +143,16 @@ class GPUWorker:
|
||||
return self.pipeline.load_weights(weights)
|
||||
|
||||
def sleep(self, level: int = 1) -> bool:
|
||||
"""
|
||||
Put the worker to sleep. The worker should not process any requests.
|
||||
The caller should guarantee that no requests are being processed
|
||||
during the sleep period, before `wake_up` is called.
|
||||
|
||||
Args:
|
||||
level: The sleep level. Level 1 sleep will offload the model
|
||||
weights and discard the kv cache.
|
||||
Currently only support level 1.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
@@ -166,6 +176,17 @@ class GPUWorker:
|
||||
return True
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
"""
|
||||
Wake up the worker from sleep mode. See the sleep function
|
||||
method for more details.
|
||||
|
||||
Args:
|
||||
tags: An optional list of tags to reallocate the worker memory
|
||||
for specific memory allocations. Values must be in
|
||||
`("weights")`. If None, all memory is reallocated.
|
||||
wake_up should be called with all tags (or None) before the
|
||||
worker is used again.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
|
||||
Reference in new issue
Block a user