[Bugfix] Support passing vllm cli args to online serving in vLLM-Omni (#206)
Signed-off-by: Gaohan123 <gaohan19@huawei.com> Signed-off-by: Gaohan123 <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
bbb5a72871
commit
45dc189d8d
5 files changed
+131
-36
No files matched your search
@@ -185,10 +185,7 @@ async def build_async_omni_from_stage_config(
|
||||
async_omni: Optional[EngineClient] = None
|
||||
|
||||
try:
|
||||
if getattr(args, "stage_configs_path", None):
|
||||
async_omni = AsyncOmni(model=args.model, stage_configs_path=args.stage_configs_path)
|
||||
else:
|
||||
async_omni = AsyncOmni(model=args.model)
|
||||
async_omni = AsyncOmni(model=args.model, cli_args=args)
|
||||
|
||||
# # Don't keep the dummy data in memory
|
||||
# await async_llm.reset_mm_cache()
|
||||
|
||||
Reference in new issue
Block a user