[Perf] Bagel KV-ready early forwarding and time step consistency for /v1/chat/completions (#2398)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
08cb436d4e
commit
d40840b114
7 files changed
+102
-37
No files matched your search
@@ -120,7 +120,6 @@ def _generate_bagel_image(
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
params_list = omni.default_sampling_params_list
|
||||
params_list[0].max_tokens = 1 # type: ignore
|
||||
if len(params_list) > 1:
|
||||
params_list[1].num_inference_steps = num_inference_steps # type: ignore
|
||||
params_list[1].extra_args = { # type: ignore
|
||||
|
||||
Reference in new issue
Block a user