[Bugfix] Fix colocate-async sleep admission race and deliver AR abort tokens (#6367)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Signed-off-by: Cursor <cursoragent@cursor.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
authored and GitHub committed 2026-08-29 09:11:39 +00:00
1 parent 5c47fd0bc8
commit bb5bf8fd0a
23 files changed
+1229 -157

No files matched your search

+9 -1
View File
@@ -201,11 +201,19 @@ class OmniGPUWorkerBase(GPUWorker):
"""
from vllm.device_allocator.cumem import CuMemAllocator
# Drain in-flight kernels before CuMem unmap. Abort/decode can still
# have GPU work after the scheduler reports idle; unmapping then
# returns CUDA_ERROR_INVALID_VALUE (cudaErrorInvalidValue).
current_omni_platform.synchronize()
gc.collect()
mem_before = current_omni_platform.get_current_memory_usage(self.device)
offload_tags = ("weights",) if level == 1 else tuple()
allocator = CuMemAllocator.get_instance()
allocator.sleep(offload_tags=offload_tags)
current_omni_platform.empty_cache()
# allocator.sleep already gc.collects and empty_cache()s. A second
# empty_cache on the pluggable allocator is a known CUDA invalid
# argument (pytorch/pytorch#145168). Only sync for the readout.
current_omni_platform.synchronize()
mem_after = current_omni_platform.get_current_memory_usage(self.device)
freed = max(0, mem_before - mem_after)