[Feature]: Support cfg kv-cache transfer in multi-stage (#1422)
Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
68764cc4f4
commit
e37a89ffcb
21 files changed
+804
-29
No files matched your search
@@ -56,7 +56,10 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar
|
||||
enable_cache_dit_summary=enable_cache_dit_summary,
|
||||
parallel_config=SimpleNamespace(use_hsdp=False),
|
||||
)
|
||||
runner.kv_transfer_manager = SimpleNamespace(receive_kv_cache=lambda req, target_device: None)
|
||||
runner.kv_transfer_manager = SimpleNamespace(
|
||||
receive_kv_cache=lambda req, target_device=None: None,
|
||||
receive_multi_kv_cache=lambda req, cfg_kv_collect_func=None, target_device=None: None,
|
||||
)
|
||||
return runner
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user