[Feature]: Support cfg kv-cache transfer in multi-stage (#1422)
Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
68764cc4f4
commit
e37a89ffcb
21 files changed
+804
-29
No files matched your search
@@ -191,7 +191,11 @@ class DiffusionModelRunner:
|
||||
grad_context = torch.no_grad() if use_hsdp else torch.inference_mode()
|
||||
with grad_context:
|
||||
# The manager handles the check for need_recv_cache internally
|
||||
self.kv_transfer_manager.receive_kv_cache(req, target_device=getattr(self.pipeline, "device", None))
|
||||
self.kv_transfer_manager.receive_multi_kv_cache(
|
||||
req,
|
||||
cfg_kv_collect_func=getattr(self.od_config, "cfg_kv_collect_func", None),
|
||||
target_device=getattr(self.pipeline, "device", None),
|
||||
)
|
||||
|
||||
if req.sampling_params.generator is None and req.sampling_params.seed is not None:
|
||||
if req.sampling_params.generator_device is not None:
|
||||
|
||||
Reference in new issue
Block a user