[feat] Support prompt embedding caching for diffusion model (#2962)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Samit <285365963@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
9 files changed
+980
-11
No files matched your search
@@ -1949,6 +1949,8 @@ class AsyncOmniEngine:
|
||||
"mp" if kwargs.get("distributed_executor_backend") is None else kwargs["distributed_executor_backend"]
|
||||
),
|
||||
"enable_sleep_mode": kwargs.get("enable_sleep_mode", False),
|
||||
"enable_prompt_embed_cache": kwargs.get("enable_prompt_embed_cache", False),
|
||||
"prompt_embed_cache_size": kwargs.get("prompt_embed_cache_size", 32),
|
||||
"enable_multithread_weight_load": kwargs.get("enable_multithread_weight_load", True),
|
||||
"num_weight_load_threads": kwargs.get("num_weight_load_threads", 4),
|
||||
"quantization": kwargs.get("quantization", None),
|
||||
|
||||
Reference in new issue
Block a user