[BugFix] Fix the issue of thinker requests being preempted, causing shape mismatch. (#3147)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
440c718d2a
commit
e7ee5de09f
5 files changed
+65
-74
No files matched your search
@@ -44,7 +44,7 @@ def get_batch_token_config(default_path):
|
||||
return modify_stage_config(
|
||||
default_path,
|
||||
updates={
|
||||
"stages": {1: {"max_num_batched_tokens": 64}},
|
||||
"stages": {0: {"max_num_batched_tokens": 64}, 1: {"max_num_batched_tokens": 64}},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -95,7 +95,12 @@ test_params = [
|
||||
|
||||
test_token_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(model=model, stage_config_path=get_batch_token_config(default_path), use_stage_cli=True),
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=get_batch_token_config(default_path),
|
||||
use_stage_cli=True,
|
||||
server_args=["--async-chunk"],
|
||||
),
|
||||
id="batch_token_64",
|
||||
)
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user