[BugFix] initial sampling_params_list when offline inference (#468)

Signed-off-by: Junhong <liujunhong11@huawei.com>
Signed-off-by: Junhong Liu <98734602+LJH-LBJ@users.noreply.github.com>
Signed-off-by: Junhong Liu <ljh_lbj@163.com>
This commit is contained in:
Junhong Liu authored and GitHub committed 2025-12-26 13:54:12 +08:00
1 parent 4f1c34099a
commit 51eefbfebc
2 files changed
+13 -55

No files matched your search

+7
View File
@@ -114,6 +114,9 @@ class OmniLLM:
self.config_path, worker_backend=self.worker_backend, shm_threshold_bytes=shm_threshold_bytes
)
self.stage_list: list[OmniStage] = []
self.default_sampling_params_list: list[SamplingParams] = []
self._initialize_stages(model, init_sleep_seconds, shm_threshold_bytes, init_timeout)
def _initialize_stages(
@@ -124,6 +127,7 @@ class OmniLLM:
init_timeout: int,
) -> None:
self.stage_list: list[OmniStage] = []
self.default_sampling_params_list: list[SamplingParams] = []
# Build OmniStage instances in parallel, preserve original order
def _build_stage(idx_cfg: tuple[int, Any]) -> tuple[int, OmniStage]:
@@ -137,6 +141,7 @@ class OmniLLM:
results.append(fut.result())
results.sort(key=lambda x: x[0])
self.stage_list = [st for _, st in results]
self.default_sampling_params_list = [st.default_sampling_params for st in self.stage_list]
self.output_modalities = [st.final_output_type for st in self.stage_list]
logger.debug("[Orchestrator] Loaded %d stages", len(self.stage_list))
@@ -243,6 +248,8 @@ class OmniLLM:
ValueError: If sampling_params_list is None or has incorrect length.
"""
try:
if sampling_params_list is None:
sampling_params_list = self.default_sampling_params_list
return self._run_generation(prompts, sampling_params_list)
except Exception as e:
logger.exception("[Orchestrator] Failed to run generation: %s", e)