Merge remote-tracking branch 'origin/main' into dev/vllm-align
Made-with: Cursor # Conflicts: # vllm_omni/entrypoints/openai/serving_chat.py # vllm_omni/worker/gpu_model_runner.py Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
3e539fcb6a
37 files changed
+1072
-169
No files matched your search
@@ -190,6 +190,8 @@ class AsyncOmni(OmniBase):
|
||||
"enforce_eager": kwargs.get("enforce_eager", False),
|
||||
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
|
||||
"custom_pipeline_args": kwargs.get("custom_pipeline_args", None),
|
||||
"enable_multithread_weight_load": kwargs.get("enable_multithread_weight_load", True),
|
||||
"num_weight_load_threads": kwargs.get("num_weight_load_threads", 4),
|
||||
},
|
||||
"final_output": True,
|
||||
"final_output_type": "image",
|
||||
|
||||
@@ -60,6 +60,7 @@ class AsyncOmniDiffusion:
|
||||
# Capture stage info from kwargs before they might be filtered out
|
||||
stage_id = kwargs.get("stage_id")
|
||||
engine_input_source = kwargs.get("engine_input_source")
|
||||
cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
|
||||
|
||||
# Build config
|
||||
if od_config is None:
|
||||
@@ -115,6 +116,9 @@ class AsyncOmniDiffusion:
|
||||
else:
|
||||
raise
|
||||
|
||||
if cfg_kv_collect_func is not None:
|
||||
od_config.cfg_kv_collect_func = cfg_kv_collect_func
|
||||
|
||||
# Initialize engine
|
||||
self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
|
||||
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
"""CFG companion request tracker for the Omni orchestrator.
|
||||
|
||||
Encapsulates all bookkeeping for Classifier-Free Guidance companion
|
||||
requests (prompt expansion, parent/companion ID mapping, completion
|
||||
tracking, deferred forwarding, failure propagation, and timeouts)
|
||||
so that ``Omni._run_generation`` stays clean.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.adapter import try_send_via_connector
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CfgCompanionTracker:
|
||||
"""Manages CFG companion request lifecycle in the orchestrator scheduling loop."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
prompt_expand_func: Callable[..., Any] | None,
|
||||
stage0_sampling_params: Any,
|
||||
timeout_s: float | None = None,
|
||||
) -> None:
|
||||
self._expand_func = prompt_expand_func
|
||||
self._sp0 = stage0_sampling_params
|
||||
self._timeout_s = (
|
||||
timeout_s if timeout_s is not None else float(os.environ.get("VLLM_CFG_PENDING_TIMEOUT_S", "120"))
|
||||
)
|
||||
|
||||
self._companion_map: dict[str, dict[str, str]] = {} # parent -> {role: companion_id}
|
||||
self._companion_ids: set[str] = set()
|
||||
self._companion_to_parent: dict[str, str] = {} # companion -> parent
|
||||
self._done: dict[str, set[str]] = {} # parent -> completed companion ids
|
||||
self._pending_parents: dict[str, dict[str, Any]] = {} # parent -> deferred result
|
||||
self._failed_parents: set[str] = set()
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
return bool(self._companion_ids)
|
||||
|
||||
@property
|
||||
def num_companions(self) -> int:
|
||||
return len(self._companion_ids)
|
||||
|
||||
@property
|
||||
def stage0_sampling_params(self) -> Any:
|
||||
return self._sp0
|
||||
|
||||
def expand_prompts(
|
||||
self,
|
||||
request_id_to_prompt: dict[str, Any],
|
||||
) -> list[tuple[str, Any]]:
|
||||
"""Expand user prompts into ``(companion_id, prompt)`` pairs via model-specific func."""
|
||||
if not self._expand_func:
|
||||
return []
|
||||
|
||||
pairs: list[tuple[str, Any]] = []
|
||||
for rid, prompt in request_id_to_prompt.items():
|
||||
expanded = self._expand_func(prompt, self._sp0)
|
||||
if not expanded:
|
||||
continue
|
||||
role_map: dict[str, str] = {}
|
||||
for ep in expanded:
|
||||
cid = f"{rid}{ep.request_id_suffix}"
|
||||
role_map[ep.role] = cid
|
||||
self._companion_ids.add(cid)
|
||||
self._companion_to_parent[cid] = rid
|
||||
pairs.append((cid, ep.prompt))
|
||||
self._companion_map[rid] = role_map
|
||||
self._done[rid] = set()
|
||||
|
||||
logger.debug(
|
||||
"CFG expansion: %d parent(s) -> %d companion(s)",
|
||||
len(self._companion_map),
|
||||
len(self._companion_ids),
|
||||
)
|
||||
return pairs
|
||||
|
||||
def is_companion(self, req_id: str) -> bool:
|
||||
return req_id in self._companion_ids
|
||||
|
||||
def has_companions(self, parent_id: str) -> bool:
|
||||
return parent_id in self._companion_map
|
||||
|
||||
def all_companions_done(self, parent_id: str) -> bool:
|
||||
role_map = self._companion_map.get(parent_id, {})
|
||||
done_set = self._done.get(parent_id, set())
|
||||
return all(cid in done_set for cid in role_map.values())
|
||||
|
||||
def get_companion_request_ids(self, parent_id: str) -> dict[str, str]:
|
||||
"""Return ``{role: companion_request_id}`` for a parent."""
|
||||
return self._companion_map.get(parent_id, {})
|
||||
|
||||
def is_parent_failed(self, parent_id: str) -> bool:
|
||||
return parent_id in self._failed_parents
|
||||
|
||||
# -- Lifecycle events --
|
||||
|
||||
def on_companion_error(self, companion_id: str) -> tuple[str | None, bool]:
|
||||
"""Record failure. Returns ``(parent_id, parent_was_aborted)``."""
|
||||
parent_id = self._companion_to_parent.get(companion_id)
|
||||
if parent_id is None:
|
||||
return None, False
|
||||
self._failed_parents.add(parent_id)
|
||||
logger.error("CFG companion %s failed; marking parent %s as failed", companion_id, parent_id)
|
||||
aborted = parent_id in self._pending_parents
|
||||
if aborted:
|
||||
self._pending_parents.pop(parent_id, None)
|
||||
return parent_id, aborted
|
||||
|
||||
def on_companion_completed(self, companion_id: str) -> str | None:
|
||||
"""Mark done. Returns parent_id only if parent is pending and all companions finished."""
|
||||
parent_id = self._companion_to_parent.get(companion_id)
|
||||
if parent_id is None:
|
||||
return None
|
||||
self._done[parent_id].add(companion_id)
|
||||
logger.debug("CFG companion %s completed (parent=%s)", companion_id, parent_id)
|
||||
if parent_id in self._pending_parents and self.all_companions_done(parent_id):
|
||||
return parent_id
|
||||
return None
|
||||
|
||||
def consume_parent_failure(self, parent_id: str) -> None:
|
||||
self._failed_parents.discard(parent_id)
|
||||
|
||||
# -- Deferred parent management --
|
||||
|
||||
def defer_parent(self, parent_id: str, engine_outputs: Any, stage_id: int) -> None:
|
||||
"""Hold parent result while waiting for companions to finish."""
|
||||
self._pending_parents[parent_id] = {
|
||||
"engine_outputs": engine_outputs,
|
||||
"stage_id": stage_id,
|
||||
"pending_since": time.time(),
|
||||
}
|
||||
logger.debug("Parent %s deferred, waiting for CFG companions", parent_id)
|
||||
|
||||
def pop_pending_parent(self, parent_id: str) -> dict[str, Any] | None:
|
||||
return self._pending_parents.pop(parent_id, None)
|
||||
|
||||
def check_timeouts(self) -> list[str]:
|
||||
"""Return and remove parent IDs that exceeded the pending timeout."""
|
||||
if not self._pending_parents:
|
||||
return []
|
||||
now = time.time()
|
||||
timed_out: list[str] = []
|
||||
for pid in list(self._pending_parents):
|
||||
pending_since = self._pending_parents[pid].get("pending_since", now)
|
||||
if now - pending_since > self._timeout_s:
|
||||
self._pending_parents.pop(pid)
|
||||
self._failed_parents.discard(pid)
|
||||
timed_out.append(pid)
|
||||
logger.error("Parent %s timed out waiting for CFG companions (>%.0fs)", pid, self._timeout_s)
|
||||
return timed_out
|
||||
|
||||
# -- Forward parent with CFG KV --
|
||||
|
||||
def forward_parent_with_cfg(
|
||||
self,
|
||||
req_id: str,
|
||||
parent_result: dict[str, Any],
|
||||
stage_list: Sequence[Any],
|
||||
connectors: dict[tuple[str, str], Any],
|
||||
sampling_params_list: Sequence[OmniSamplingParams],
|
||||
request_id_to_prompt: dict[str, Any],
|
||||
final_stage_id_to_prompt: dict[str, int],
|
||||
metrics: Any,
|
||||
remaining_by_stage: list[int],
|
||||
) -> bool:
|
||||
"""Forward a parent request to the next stage with CFG KV request IDs attached."""
|
||||
stage_id = parent_result["stage_id"]
|
||||
next_stage_id = stage_id + 1
|
||||
if next_stage_id > final_stage_id_to_prompt.get(req_id, 0):
|
||||
return True
|
||||
|
||||
next_stage = stage_list[next_stage_id]
|
||||
try:
|
||||
with metrics.stage_postprocess_timer(stage_id, req_id):
|
||||
next_inputs = next_stage.process_engine_inputs(
|
||||
stage_list,
|
||||
[request_id_to_prompt[req_id]],
|
||||
source_outputs_override=parent_result["engine_outputs"],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Process engine inputs error for req %s at stage %d: %s",
|
||||
req_id,
|
||||
next_stage_id,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
sp_next = copy.deepcopy(sampling_params_list[next_stage_id])
|
||||
if isinstance(sp_next, OmniDiffusionSamplingParams):
|
||||
sp_next.cfg_kv_request_ids = self.get_companion_request_ids(req_id)
|
||||
logger.info(
|
||||
"Attaching cfg_kv_request_ids=%s to request %s",
|
||||
sp_next.cfg_kv_request_ids,
|
||||
req_id,
|
||||
)
|
||||
|
||||
connector_key = (str(stage_id), str(next_stage_id))
|
||||
connector = connectors.get(connector_key)
|
||||
sent_via_connector = False
|
||||
if connector:
|
||||
sent_via_connector = try_send_via_connector(
|
||||
connector=connector,
|
||||
stage_id=stage_id,
|
||||
next_stage_id=next_stage_id,
|
||||
req_id=req_id,
|
||||
next_inputs=next_inputs,
|
||||
sampling_params=sp_next,
|
||||
original_prompt=request_id_to_prompt[req_id],
|
||||
next_stage_queue_submit_fn=stage_list[next_stage_id].submit,
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
if not sent_via_connector:
|
||||
raise RuntimeError(
|
||||
f"Failed to send CFG request {req_id} to stage-{next_stage_id} via connector. "
|
||||
"Configure a connector for this edge or inspect connector logs for details."
|
||||
)
|
||||
|
||||
logger.debug("Forwarded CFG-enabled request %s to stage-%d", req_id, next_stage_id)
|
||||
remaining_by_stage[next_stage_id] += 1
|
||||
return True
|
||||
@@ -272,6 +272,21 @@ class OmniServeCommand(CLISubcommand):
|
||||
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
|
||||
)
|
||||
|
||||
# Parallel weight loading (faster diffusion startup)
|
||||
omni_config_group.add_argument(
|
||||
"--disable-multithread-weight-load",
|
||||
action="store_false",
|
||||
dest="enable_multithread_weight_load",
|
||||
default=True,
|
||||
help="Disable multi-threaded safetensors loading (default: enabled with 4 threads).",
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--num-weight-load-threads",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Number of threads for parallel weight loading (default: 4).",
|
||||
)
|
||||
|
||||
# diffusion model offload parameters
|
||||
omni_config_group.add_argument(
|
||||
"--enable-cpu-offload",
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm_omni.distributed.ray_utils.utils import (
|
||||
get_ray_queue_class,
|
||||
try_close_ray,
|
||||
)
|
||||
from vllm_omni.entrypoints.cfg_companion_tracker import CfgCompanionTracker
|
||||
from vllm_omni.entrypoints.omni_stage import OmniStage
|
||||
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK, OmniStageTaskType
|
||||
from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
|
||||
@@ -935,6 +936,13 @@ class Omni(OmniBase):
|
||||
_req_start_ts: dict[str, float] = {}
|
||||
_wall_start_ts: float = time.time()
|
||||
|
||||
# CFG companion tracking (prompt expansion + lifecycle management)
|
||||
cfg = CfgCompanionTracker(
|
||||
prompt_expand_func=getattr(self.stage_list[0], "prompt_expand_func", None),
|
||||
stage0_sampling_params=sampling_params_list[0],
|
||||
)
|
||||
expanded_companions = cfg.expand_prompts(request_id_to_prompt)
|
||||
|
||||
# Determine the final stage for E2E stats (highest stage_id with final_output=True; fallback to last stage)
|
||||
final_stage_id_to_prompt: dict[str, int] = {}
|
||||
for rid, prompt in request_id_to_prompt.items():
|
||||
@@ -976,6 +984,18 @@ class Omni(OmniBase):
|
||||
_req_start_ts[req_id] = time.time()
|
||||
logger.debug(f"[{self._name}] Enqueued request {req_id} to stage-0")
|
||||
|
||||
# Submit CFG companion requests to stage-0
|
||||
if cfg.is_active:
|
||||
for companion_id, companion_prompt in expanded_companions:
|
||||
task = {
|
||||
"request_id": companion_id,
|
||||
"engine_inputs": companion_prompt,
|
||||
"sampling_params": cfg.stage0_sampling_params,
|
||||
}
|
||||
self.stage_list[0].submit(task)
|
||||
_req_start_ts[companion_id] = time.time()
|
||||
logger.debug(f"[{self._name}] Enqueued CFG companion {companion_id} to stage-0")
|
||||
|
||||
pbar = None
|
||||
if use_tqdm:
|
||||
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
|
||||
@@ -987,7 +1007,7 @@ class Omni(OmniBase):
|
||||
)
|
||||
# For each stage, forward results to next stage; collect finals at the end
|
||||
# We pipeline by continually polling output queues in stage order
|
||||
remaining_by_stage: list[int] = [len(request_prompts)] + [0] * (num_stages - 1)
|
||||
remaining_by_stage: list[int] = [len(request_prompts) + cfg.num_companions] + [0] * (num_stages - 1)
|
||||
completed_requests = 0
|
||||
total_requests = len(request_prompts)
|
||||
|
||||
@@ -1007,6 +1027,14 @@ class Omni(OmniBase):
|
||||
logger.error(
|
||||
f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
|
||||
)
|
||||
if cfg.is_companion(req_id) and stage_id == 0:
|
||||
parent_id, parent_aborted = cfg.on_companion_error(req_id)
|
||||
if parent_aborted:
|
||||
completed_requests += 1
|
||||
logger.error(
|
||||
f"[{self._name}] Parent {parent_id} aborted due to "
|
||||
f"companion failure ({completed_requests}/{total_requests})",
|
||||
)
|
||||
continue
|
||||
|
||||
if result.get("type") == "stage_ready":
|
||||
@@ -1015,6 +1043,30 @@ class Omni(OmniBase):
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
|
||||
# CFG: companion requests only run through Stage-0
|
||||
if cfg.is_companion(req_id) and stage_id == 0:
|
||||
ready_parent = cfg.on_companion_completed(req_id)
|
||||
if ready_parent is not None:
|
||||
success = cfg.forward_parent_with_cfg(
|
||||
ready_parent,
|
||||
cfg.pop_pending_parent(ready_parent),
|
||||
self.stage_list,
|
||||
self.connectors,
|
||||
sampling_params_list,
|
||||
request_id_to_prompt,
|
||||
final_stage_id_to_prompt,
|
||||
metrics,
|
||||
remaining_by_stage,
|
||||
)
|
||||
if not success:
|
||||
cfg.consume_parent_failure(ready_parent)
|
||||
completed_requests += 1
|
||||
logger.error(
|
||||
f"[{self._name}] Parent {ready_parent} dropped due to CFG forwarding failure "
|
||||
f"({completed_requests}/{total_requests})",
|
||||
)
|
||||
continue
|
||||
|
||||
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
|
||||
# Mark last output time for this stage whenever we receive outputs
|
||||
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
|
||||
@@ -1104,6 +1156,40 @@ class Omni(OmniBase):
|
||||
|
||||
next_stage_id = stage_id + 1
|
||||
if next_stage_id <= final_stage_id_to_prompt[req_id]:
|
||||
# CFG: if this parent has companions, defer forwarding
|
||||
if cfg.has_companions(req_id) and stage_id == 0:
|
||||
if cfg.is_parent_failed(req_id):
|
||||
cfg.consume_parent_failure(req_id)
|
||||
completed_requests += 1
|
||||
logger.error(
|
||||
f"[{self._name}] Parent {req_id} skipped CFG forwarding due to "
|
||||
f"companion failure ({completed_requests}/{total_requests})",
|
||||
)
|
||||
continue
|
||||
|
||||
if cfg.all_companions_done(req_id):
|
||||
success = cfg.forward_parent_with_cfg(
|
||||
req_id,
|
||||
{"engine_outputs": engine_outputs, "stage_id": stage_id},
|
||||
self.stage_list,
|
||||
self.connectors,
|
||||
sampling_params_list,
|
||||
request_id_to_prompt,
|
||||
final_stage_id_to_prompt,
|
||||
metrics,
|
||||
remaining_by_stage,
|
||||
)
|
||||
if not success:
|
||||
cfg.consume_parent_failure(req_id)
|
||||
completed_requests += 1
|
||||
logger.error(
|
||||
f"[{self._name}] Parent {req_id} dropped due to CFG forwarding failure "
|
||||
f"({completed_requests}/{total_requests})",
|
||||
)
|
||||
else:
|
||||
cfg.defer_parent(req_id, engine_outputs, stage_id)
|
||||
continue
|
||||
|
||||
next_stage: OmniStage = self.stage_list[next_stage_id]
|
||||
try:
|
||||
# Derive inputs for the next stage, record preprocess time
|
||||
@@ -1112,9 +1198,10 @@ class Omni(OmniBase):
|
||||
self.stage_list, [request_id_to_prompt[req_id]]
|
||||
)
|
||||
except Exception as e:
|
||||
completed_requests += 1
|
||||
logger.exception(
|
||||
f"[{self._name}] Process engine inputs error for req {req_id}"
|
||||
f" at stage {next_stage_id}: {e}",
|
||||
f" at stage {next_stage_id}: {e} ({completed_requests}/{total_requests})",
|
||||
)
|
||||
continue
|
||||
sp_next = sampling_params_list[next_stage_id] # type: ignore[index]
|
||||
@@ -1141,6 +1228,7 @@ class Omni(OmniBase):
|
||||
f"[{self._name}] Failed to send request {req_id} to stage-{next_stage_id} via connector. "
|
||||
"Configure a connector for this edge or inspect connector logs for details."
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"[{self._name}] Forwarded request {req_id} to stage-{next_stage_id}",
|
||||
)
|
||||
@@ -1154,6 +1242,12 @@ class Omni(OmniBase):
|
||||
logger.debug(
|
||||
f"[{self._name}] Request {req_id} fully completed ({completed_requests}/{total_requests})",
|
||||
)
|
||||
for timed_out_id in cfg.check_timeouts():
|
||||
completed_requests += 1
|
||||
logger.error(
|
||||
f"[{self._name}] Parent {timed_out_id} timed out; counting as failed "
|
||||
f"({completed_requests}/{total_requests})",
|
||||
)
|
||||
|
||||
if not made_progress:
|
||||
time.sleep(0.005)
|
||||
|
||||
@@ -28,6 +28,7 @@ class OmniDiffusion:
|
||||
# Capture stage info from kwargs before they might be filtered out
|
||||
stage_id = kwargs.get("stage_id")
|
||||
engine_input_source = kwargs.get("engine_input_source")
|
||||
cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
|
||||
|
||||
if od_config is None:
|
||||
od_config = OmniDiffusionConfig.from_kwargs(**kwargs)
|
||||
@@ -94,6 +95,9 @@ class OmniDiffusion:
|
||||
od_config.tf_model_config = TransformerConfig()
|
||||
od_config.update_multimodal_support()
|
||||
|
||||
if cfg_kv_collect_func is not None:
|
||||
od_config.cfg_kv_collect_func = cfg_kv_collect_func
|
||||
|
||||
self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
|
||||
|
||||
def generate(
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm_omni.entrypoints.stage_utils import (
|
||||
_resolve_model_tokenizer_paths,
|
||||
_to_dict,
|
||||
is_profiler_task,
|
||||
load_func_from_config,
|
||||
maybe_dump_to_shm,
|
||||
set_stage_devices,
|
||||
)
|
||||
@@ -280,6 +281,7 @@ class OmniStage:
|
||||
else:
|
||||
self.custom_process_input_func = None
|
||||
|
||||
self.prompt_expand_func = load_func_from_config(getattr(stage_config, "prompt_expand_func", None))
|
||||
self.final_output = getattr(stage_config, "final_output", False)
|
||||
self.final_output_type = getattr(stage_config, "final_output_type", None)
|
||||
self.tts_args = _to_dict(getattr(stage_config, "tts_args", {}))
|
||||
@@ -460,6 +462,7 @@ class OmniStage:
|
||||
"connectors_config": connectors_config or {},
|
||||
"stage_type": self.stage_type,
|
||||
"engine_input_source": self.engine_input_source,
|
||||
"cfg_kv_collect_func": getattr(self.stage_config, "cfg_kv_collect_func", None),
|
||||
"final_output": self.final_output,
|
||||
"final_output_type": self.final_output_type,
|
||||
}
|
||||
@@ -612,7 +615,11 @@ class OmniStage:
|
||||
return None
|
||||
|
||||
def process_engine_inputs(
|
||||
self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None
|
||||
self,
|
||||
stage_list: list[Any],
|
||||
prompt: OmniTokensPrompt | TextPrompt = None,
|
||||
*,
|
||||
source_outputs_override: Any = None,
|
||||
) -> list[OmniTokensPrompt | TextPrompt]:
|
||||
"""Process engine inputs for this stage from upstream stage outputs.
|
||||
|
||||
@@ -623,6 +630,8 @@ class OmniStage:
|
||||
Args:
|
||||
stage_list: List of all stages in the pipeline
|
||||
prompt: Optional original prompt (for multimodal data preservation)
|
||||
source_outputs_override: Use these outputs instead of reading from
|
||||
the source stage's ``engine_outputs`` (for deferred CFG requests).
|
||||
|
||||
Returns:
|
||||
List of processed engine inputs ready for this stage
|
||||
@@ -635,7 +644,11 @@ class OmniStage:
|
||||
if len(self.engine_input_source) == 0:
|
||||
raise ValueError("engine_input_source is empty")
|
||||
source_stage_id = self.engine_input_source[0]
|
||||
source_outputs = stage_list[source_stage_id].engine_outputs
|
||||
source_outputs = (
|
||||
source_outputs_override
|
||||
if source_outputs_override is not None
|
||||
else stage_list[source_stage_id].engine_outputs
|
||||
)
|
||||
if not isinstance(prompt, list):
|
||||
prompt = [prompt]
|
||||
multi_modal_data = {
|
||||
@@ -657,6 +670,18 @@ class OmniStage:
|
||||
|
||||
else:
|
||||
engine_input_source = self.engine_input_source
|
||||
if source_outputs_override is not None and engine_input_source:
|
||||
# Temporarily swap engine_outputs so custom_process_input_func
|
||||
# (which reads stage_list directly) sees the correct data.
|
||||
_source_id = engine_input_source[0]
|
||||
_orig_outputs = stage_list[_source_id].engine_outputs
|
||||
stage_list[_source_id].engine_outputs = source_outputs_override
|
||||
try:
|
||||
return self.custom_process_input_func(
|
||||
stage_list, engine_input_source, prompt, self.requires_multimodal_data
|
||||
)
|
||||
finally:
|
||||
stage_list[_source_id].engine_outputs = _orig_outputs
|
||||
return self.custom_process_input_func(
|
||||
stage_list, engine_input_source, prompt, self.requires_multimodal_data
|
||||
)
|
||||
@@ -702,6 +727,8 @@ def _stage_worker(
|
||||
connectors_config = stage_payload.get("connectors_config", {})
|
||||
stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm")
|
||||
|
||||
cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
|
||||
|
||||
if stage_type != "diffusion":
|
||||
_resolve_worker_cls(engine_args)
|
||||
|
||||
@@ -761,6 +788,7 @@ def _stage_worker(
|
||||
model=model,
|
||||
stage_id=stage_id,
|
||||
engine_input_source=stage_payload.get("engine_input_source", []),
|
||||
cfg_kv_collect_func=cfg_kv_collect_func,
|
||||
**engine_args,
|
||||
)
|
||||
else:
|
||||
@@ -1109,6 +1137,7 @@ async def _stage_worker_async(
|
||||
final_output = stage_payload.get("final_output", False)
|
||||
final_output_type = stage_payload.get("final_output_type", None)
|
||||
|
||||
cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
|
||||
# Handle non-standard model directory structures (e.g., tokenizer in root, model in subdir)
|
||||
model = _resolve_model_tokenizer_paths(model, engine_args)
|
||||
|
||||
@@ -1187,10 +1216,13 @@ async def _stage_worker_async(
|
||||
od_config["omni_kv_config"]["engine_input_source"] = stage_payload.get("engine_input_source", [])
|
||||
|
||||
logger.debug(f"[Stage-%s] Initializing diffusion engine with config: {od_config}", stage_id)
|
||||
_diffusion_kwargs = {k: v for k, v in engine_args.items() if k not in {"od_config", "model"}}
|
||||
if cfg_kv_collect_func is not None:
|
||||
_diffusion_kwargs["cfg_kv_collect_func"] = cfg_kv_collect_func
|
||||
stage_engine = AsyncOmniDiffusion(
|
||||
model=model,
|
||||
od_config=od_config,
|
||||
**{k: v for k, v in engine_args.items() if k not in {"od_config", "model"}},
|
||||
**_diffusion_kwargs,
|
||||
)
|
||||
vllm_config = None # Diffusion doesn't use vllm_config
|
||||
else:
|
||||
|
||||
@@ -13,7 +13,6 @@ import torch
|
||||
from fastapi import Request
|
||||
from PIL import Image
|
||||
from pydantic import TypeAdapter
|
||||
from vllm.renderers import BaseRenderer
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse
|
||||
@@ -67,7 +66,7 @@ from vllm.inputs.data import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.renderers import merge_kwargs
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from multiprocessing import shared_memory as _shm
|
||||
from typing import Any
|
||||
|
||||
@@ -12,6 +14,22 @@ from omegaconf import OmegaConf
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_func_from_config(func_path: str | None) -> Callable[..., Any] | None:
|
||||
"""Dynamically import a callable from a fully-qualified dotted path.
|
||||
|
||||
Args:
|
||||
func_path: Dotted path such as ``"pkg.module.func_name"``, or *None*.
|
||||
|
||||
Returns:
|
||||
The imported callable, or *None* when *func_path* is falsy.
|
||||
"""
|
||||
if not func_path:
|
||||
return None
|
||||
module_path, func_name = func_path.rsplit(".", 1)
|
||||
module = importlib.import_module(module_path)
|
||||
return getattr(module, func_name)
|
||||
|
||||
|
||||
class OmniStageTaskType(enum.Enum):
|
||||
GENERATE = "generate"
|
||||
ABORT = "abort"
|
||||
|
||||
Reference in new issue
Block a user