Merge remote-tracking branch 'origin/main' into dev/vllm-align

Made-with: Cursor

# Conflicts:
#	vllm_omni/entrypoints/openai/serving_chat.py
#	vllm_omni/worker/gpu_model_runner.py

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-03-03 02:57:27 +00:00
commit 3e539fcb6a
37 files changed
+1072 -169

No files matched your search

+2
View File
@@ -190,6 +190,8 @@ class AsyncOmni(OmniBase):
"enforce_eager": kwargs.get("enforce_eager", False),
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
"custom_pipeline_args": kwargs.get("custom_pipeline_args", None),
"enable_multithread_weight_load": kwargs.get("enable_multithread_weight_load", True),
"num_weight_load_threads": kwargs.get("num_weight_load_threads", 4),
},
"final_output": True,
"final_output_type": "image",
@@ -60,6 +60,7 @@ class AsyncOmniDiffusion:
# Capture stage info from kwargs before they might be filtered out
stage_id = kwargs.get("stage_id")
engine_input_source = kwargs.get("engine_input_source")
cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
# Build config
if od_config is None:
@@ -115,6 +116,9 @@ class AsyncOmniDiffusion:
else:
raise
if cfg_kv_collect_func is not None:
od_config.cfg_kv_collect_func = cfg_kv_collect_func
# Initialize engine
self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
@@ -0,0 +1,233 @@
"""CFG companion request tracker for the Omni orchestrator.
Encapsulates all bookkeeping for Classifier-Free Guidance companion
requests (prompt expansion, parent/companion ID mapping, completion
tracking, deferred forwarding, failure propagation, and timeouts)
so that ``Omni._run_generation`` stays clean.
"""
from __future__ import annotations
import copy
import logging
import os
import time
from collections.abc import Callable, Sequence
from typing import Any
from vllm_omni.distributed.omni_connectors.adapter import try_send_via_connector
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams
logger = logging.getLogger(__name__)
class CfgCompanionTracker:
"""Manages CFG companion request lifecycle in the orchestrator scheduling loop."""
def __init__(
self,
prompt_expand_func: Callable[..., Any] | None,
stage0_sampling_params: Any,
timeout_s: float | None = None,
) -> None:
self._expand_func = prompt_expand_func
self._sp0 = stage0_sampling_params
self._timeout_s = (
timeout_s if timeout_s is not None else float(os.environ.get("VLLM_CFG_PENDING_TIMEOUT_S", "120"))
)
self._companion_map: dict[str, dict[str, str]] = {} # parent -> {role: companion_id}
self._companion_ids: set[str] = set()
self._companion_to_parent: dict[str, str] = {} # companion -> parent
self._done: dict[str, set[str]] = {} # parent -> completed companion ids
self._pending_parents: dict[str, dict[str, Any]] = {} # parent -> deferred result
self._failed_parents: set[str] = set()
@property
def is_active(self) -> bool:
return bool(self._companion_ids)
@property
def num_companions(self) -> int:
return len(self._companion_ids)
@property
def stage0_sampling_params(self) -> Any:
return self._sp0
def expand_prompts(
self,
request_id_to_prompt: dict[str, Any],
) -> list[tuple[str, Any]]:
"""Expand user prompts into ``(companion_id, prompt)`` pairs via model-specific func."""
if not self._expand_func:
return []
pairs: list[tuple[str, Any]] = []
for rid, prompt in request_id_to_prompt.items():
expanded = self._expand_func(prompt, self._sp0)
if not expanded:
continue
role_map: dict[str, str] = {}
for ep in expanded:
cid = f"{rid}{ep.request_id_suffix}"
role_map[ep.role] = cid
self._companion_ids.add(cid)
self._companion_to_parent[cid] = rid
pairs.append((cid, ep.prompt))
self._companion_map[rid] = role_map
self._done[rid] = set()
logger.debug(
"CFG expansion: %d parent(s) -> %d companion(s)",
len(self._companion_map),
len(self._companion_ids),
)
return pairs
def is_companion(self, req_id: str) -> bool:
return req_id in self._companion_ids
def has_companions(self, parent_id: str) -> bool:
return parent_id in self._companion_map
def all_companions_done(self, parent_id: str) -> bool:
role_map = self._companion_map.get(parent_id, {})
done_set = self._done.get(parent_id, set())
return all(cid in done_set for cid in role_map.values())
def get_companion_request_ids(self, parent_id: str) -> dict[str, str]:
"""Return ``{role: companion_request_id}`` for a parent."""
return self._companion_map.get(parent_id, {})
def is_parent_failed(self, parent_id: str) -> bool:
return parent_id in self._failed_parents
# -- Lifecycle events --
def on_companion_error(self, companion_id: str) -> tuple[str | None, bool]:
"""Record failure. Returns ``(parent_id, parent_was_aborted)``."""
parent_id = self._companion_to_parent.get(companion_id)
if parent_id is None:
return None, False
self._failed_parents.add(parent_id)
logger.error("CFG companion %s failed; marking parent %s as failed", companion_id, parent_id)
aborted = parent_id in self._pending_parents
if aborted:
self._pending_parents.pop(parent_id, None)
return parent_id, aborted
def on_companion_completed(self, companion_id: str) -> str | None:
"""Mark done. Returns parent_id only if parent is pending and all companions finished."""
parent_id = self._companion_to_parent.get(companion_id)
if parent_id is None:
return None
self._done[parent_id].add(companion_id)
logger.debug("CFG companion %s completed (parent=%s)", companion_id, parent_id)
if parent_id in self._pending_parents and self.all_companions_done(parent_id):
return parent_id
return None
def consume_parent_failure(self, parent_id: str) -> None:
self._failed_parents.discard(parent_id)
# -- Deferred parent management --
def defer_parent(self, parent_id: str, engine_outputs: Any, stage_id: int) -> None:
"""Hold parent result while waiting for companions to finish."""
self._pending_parents[parent_id] = {
"engine_outputs": engine_outputs,
"stage_id": stage_id,
"pending_since": time.time(),
}
logger.debug("Parent %s deferred, waiting for CFG companions", parent_id)
def pop_pending_parent(self, parent_id: str) -> dict[str, Any] | None:
return self._pending_parents.pop(parent_id, None)
def check_timeouts(self) -> list[str]:
"""Return and remove parent IDs that exceeded the pending timeout."""
if not self._pending_parents:
return []
now = time.time()
timed_out: list[str] = []
for pid in list(self._pending_parents):
pending_since = self._pending_parents[pid].get("pending_since", now)
if now - pending_since > self._timeout_s:
self._pending_parents.pop(pid)
self._failed_parents.discard(pid)
timed_out.append(pid)
logger.error("Parent %s timed out waiting for CFG companions (>%.0fs)", pid, self._timeout_s)
return timed_out
# -- Forward parent with CFG KV --
def forward_parent_with_cfg(
self,
req_id: str,
parent_result: dict[str, Any],
stage_list: Sequence[Any],
connectors: dict[tuple[str, str], Any],
sampling_params_list: Sequence[OmniSamplingParams],
request_id_to_prompt: dict[str, Any],
final_stage_id_to_prompt: dict[str, int],
metrics: Any,
remaining_by_stage: list[int],
) -> bool:
"""Forward a parent request to the next stage with CFG KV request IDs attached."""
stage_id = parent_result["stage_id"]
next_stage_id = stage_id + 1
if next_stage_id > final_stage_id_to_prompt.get(req_id, 0):
return True
next_stage = stage_list[next_stage_id]
try:
with metrics.stage_postprocess_timer(stage_id, req_id):
next_inputs = next_stage.process_engine_inputs(
stage_list,
[request_id_to_prompt[req_id]],
source_outputs_override=parent_result["engine_outputs"],
)
except Exception as e:
logger.exception(
"Process engine inputs error for req %s at stage %d: %s",
req_id,
next_stage_id,
e,
)
return False
sp_next = copy.deepcopy(sampling_params_list[next_stage_id])
if isinstance(sp_next, OmniDiffusionSamplingParams):
sp_next.cfg_kv_request_ids = self.get_companion_request_ids(req_id)
logger.info(
"Attaching cfg_kv_request_ids=%s to request %s",
sp_next.cfg_kv_request_ids,
req_id,
)
connector_key = (str(stage_id), str(next_stage_id))
connector = connectors.get(connector_key)
sent_via_connector = False
if connector:
sent_via_connector = try_send_via_connector(
connector=connector,
stage_id=stage_id,
next_stage_id=next_stage_id,
req_id=req_id,
next_inputs=next_inputs,
sampling_params=sp_next,
original_prompt=request_id_to_prompt[req_id],
next_stage_queue_submit_fn=stage_list[next_stage_id].submit,
metrics=metrics,
)
if not sent_via_connector:
raise RuntimeError(
f"Failed to send CFG request {req_id} to stage-{next_stage_id} via connector. "
"Configure a connector for this edge or inspect connector logs for details."
)
logger.debug("Forwarded CFG-enabled request %s to stage-%d", req_id, next_stage_id)
remaining_by_stage[next_stage_id] += 1
return True
+15
View File
@@ -272,6 +272,21 @@ class OmniServeCommand(CLISubcommand):
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
)
# Parallel weight loading (faster diffusion startup)
omni_config_group.add_argument(
"--disable-multithread-weight-load",
action="store_false",
dest="enable_multithread_weight_load",
default=True,
help="Disable multi-threaded safetensors loading (default: enabled with 4 threads).",
)
omni_config_group.add_argument(
"--num-weight-load-threads",
type=int,
default=4,
help="Number of threads for parallel weight loading (default: 4).",
)
# diffusion model offload parameters
omni_config_group.add_argument(
"--enable-cpu-offload",
+96 -2
View File
@@ -34,6 +34,7 @@ from vllm_omni.distributed.ray_utils.utils import (
get_ray_queue_class,
try_close_ray,
)
from vllm_omni.entrypoints.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.entrypoints.omni_stage import OmniStage
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK, OmniStageTaskType
from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
@@ -935,6 +936,13 @@ class Omni(OmniBase):
_req_start_ts: dict[str, float] = {}
_wall_start_ts: float = time.time()
# CFG companion tracking (prompt expansion + lifecycle management)
cfg = CfgCompanionTracker(
prompt_expand_func=getattr(self.stage_list[0], "prompt_expand_func", None),
stage0_sampling_params=sampling_params_list[0],
)
expanded_companions = cfg.expand_prompts(request_id_to_prompt)
# Determine the final stage for E2E stats (highest stage_id with final_output=True; fallback to last stage)
final_stage_id_to_prompt: dict[str, int] = {}
for rid, prompt in request_id_to_prompt.items():
@@ -976,6 +984,18 @@ class Omni(OmniBase):
_req_start_ts[req_id] = time.time()
logger.debug(f"[{self._name}] Enqueued request {req_id} to stage-0")
# Submit CFG companion requests to stage-0
if cfg.is_active:
for companion_id, companion_prompt in expanded_companions:
task = {
"request_id": companion_id,
"engine_inputs": companion_prompt,
"sampling_params": cfg.stage0_sampling_params,
}
self.stage_list[0].submit(task)
_req_start_ts[companion_id] = time.time()
logger.debug(f"[{self._name}] Enqueued CFG companion {companion_id} to stage-0")
pbar = None
if use_tqdm:
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
@@ -987,7 +1007,7 @@ class Omni(OmniBase):
)
# For each stage, forward results to next stage; collect finals at the end
# We pipeline by continually polling output queues in stage order
remaining_by_stage: list[int] = [len(request_prompts)] + [0] * (num_stages - 1)
remaining_by_stage: list[int] = [len(request_prompts) + cfg.num_companions] + [0] * (num_stages - 1)
completed_requests = 0
total_requests = len(request_prompts)
@@ -1007,6 +1027,14 @@ class Omni(OmniBase):
logger.error(
f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
)
if cfg.is_companion(req_id) and stage_id == 0:
parent_id, parent_aborted = cfg.on_companion_error(req_id)
if parent_aborted:
completed_requests += 1
logger.error(
f"[{self._name}] Parent {parent_id} aborted due to "
f"companion failure ({completed_requests}/{total_requests})",
)
continue
if result.get("type") == "stage_ready":
@@ -1015,6 +1043,30 @@ class Omni(OmniBase):
time.sleep(0.05)
continue
# CFG: companion requests only run through Stage-0
if cfg.is_companion(req_id) and stage_id == 0:
ready_parent = cfg.on_companion_completed(req_id)
if ready_parent is not None:
success = cfg.forward_parent_with_cfg(
ready_parent,
cfg.pop_pending_parent(ready_parent),
self.stage_list,
self.connectors,
sampling_params_list,
request_id_to_prompt,
final_stage_id_to_prompt,
metrics,
remaining_by_stage,
)
if not success:
cfg.consume_parent_failure(ready_parent)
completed_requests += 1
logger.error(
f"[{self._name}] Parent {ready_parent} dropped due to CFG forwarding failure "
f"({completed_requests}/{total_requests})",
)
continue
engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
# Mark last output time for this stage whenever we receive outputs
metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
@@ -1104,6 +1156,40 @@ class Omni(OmniBase):
next_stage_id = stage_id + 1
if next_stage_id <= final_stage_id_to_prompt[req_id]:
# CFG: if this parent has companions, defer forwarding
if cfg.has_companions(req_id) and stage_id == 0:
if cfg.is_parent_failed(req_id):
cfg.consume_parent_failure(req_id)
completed_requests += 1
logger.error(
f"[{self._name}] Parent {req_id} skipped CFG forwarding due to "
f"companion failure ({completed_requests}/{total_requests})",
)
continue
if cfg.all_companions_done(req_id):
success = cfg.forward_parent_with_cfg(
req_id,
{"engine_outputs": engine_outputs, "stage_id": stage_id},
self.stage_list,
self.connectors,
sampling_params_list,
request_id_to_prompt,
final_stage_id_to_prompt,
metrics,
remaining_by_stage,
)
if not success:
cfg.consume_parent_failure(req_id)
completed_requests += 1
logger.error(
f"[{self._name}] Parent {req_id} dropped due to CFG forwarding failure "
f"({completed_requests}/{total_requests})",
)
else:
cfg.defer_parent(req_id, engine_outputs, stage_id)
continue
next_stage: OmniStage = self.stage_list[next_stage_id]
try:
# Derive inputs for the next stage, record preprocess time
@@ -1112,9 +1198,10 @@ class Omni(OmniBase):
self.stage_list, [request_id_to_prompt[req_id]]
)
except Exception as e:
completed_requests += 1
logger.exception(
f"[{self._name}] Process engine inputs error for req {req_id}"
f" at stage {next_stage_id}: {e}",
f" at stage {next_stage_id}: {e} ({completed_requests}/{total_requests})",
)
continue
sp_next = sampling_params_list[next_stage_id] # type: ignore[index]
@@ -1141,6 +1228,7 @@ class Omni(OmniBase):
f"[{self._name}] Failed to send request {req_id} to stage-{next_stage_id} via connector. "
"Configure a connector for this edge or inspect connector logs for details."
)
logger.debug(
f"[{self._name}] Forwarded request {req_id} to stage-{next_stage_id}",
)
@@ -1154,6 +1242,12 @@ class Omni(OmniBase):
logger.debug(
f"[{self._name}] Request {req_id} fully completed ({completed_requests}/{total_requests})",
)
for timed_out_id in cfg.check_timeouts():
completed_requests += 1
logger.error(
f"[{self._name}] Parent {timed_out_id} timed out; counting as failed "
f"({completed_requests}/{total_requests})",
)
if not made_progress:
time.sleep(0.005)
+4
View File
@@ -28,6 +28,7 @@ class OmniDiffusion:
# Capture stage info from kwargs before they might be filtered out
stage_id = kwargs.get("stage_id")
engine_input_source = kwargs.get("engine_input_source")
cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
if od_config is None:
od_config = OmniDiffusionConfig.from_kwargs(**kwargs)
@@ -94,6 +95,9 @@ class OmniDiffusion:
od_config.tf_model_config = TransformerConfig()
od_config.update_multimodal_support()
if cfg_kv_collect_func is not None:
od_config.cfg_kv_collect_func = cfg_kv_collect_func
self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
def generate(
+35 -3
View File
@@ -47,6 +47,7 @@ from vllm_omni.entrypoints.stage_utils import (
_resolve_model_tokenizer_paths,
_to_dict,
is_profiler_task,
load_func_from_config,
maybe_dump_to_shm,
set_stage_devices,
)
@@ -280,6 +281,7 @@ class OmniStage:
else:
self.custom_process_input_func = None
self.prompt_expand_func = load_func_from_config(getattr(stage_config, "prompt_expand_func", None))
self.final_output = getattr(stage_config, "final_output", False)
self.final_output_type = getattr(stage_config, "final_output_type", None)
self.tts_args = _to_dict(getattr(stage_config, "tts_args", {}))
@@ -460,6 +462,7 @@ class OmniStage:
"connectors_config": connectors_config or {},
"stage_type": self.stage_type,
"engine_input_source": self.engine_input_source,
"cfg_kv_collect_func": getattr(self.stage_config, "cfg_kv_collect_func", None),
"final_output": self.final_output,
"final_output_type": self.final_output_type,
}
@@ -612,7 +615,11 @@ class OmniStage:
return None
def process_engine_inputs(
self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None
self,
stage_list: list[Any],
prompt: OmniTokensPrompt | TextPrompt = None,
*,
source_outputs_override: Any = None,
) -> list[OmniTokensPrompt | TextPrompt]:
"""Process engine inputs for this stage from upstream stage outputs.
@@ -623,6 +630,8 @@ class OmniStage:
Args:
stage_list: List of all stages in the pipeline
prompt: Optional original prompt (for multimodal data preservation)
source_outputs_override: Use these outputs instead of reading from
the source stage's ``engine_outputs`` (for deferred CFG requests).
Returns:
List of processed engine inputs ready for this stage
@@ -635,7 +644,11 @@ class OmniStage:
if len(self.engine_input_source) == 0:
raise ValueError("engine_input_source is empty")
source_stage_id = self.engine_input_source[0]
source_outputs = stage_list[source_stage_id].engine_outputs
source_outputs = (
source_outputs_override
if source_outputs_override is not None
else stage_list[source_stage_id].engine_outputs
)
if not isinstance(prompt, list):
prompt = [prompt]
multi_modal_data = {
@@ -657,6 +670,18 @@ class OmniStage:
else:
engine_input_source = self.engine_input_source
if source_outputs_override is not None and engine_input_source:
# Temporarily swap engine_outputs so custom_process_input_func
# (which reads stage_list directly) sees the correct data.
_source_id = engine_input_source[0]
_orig_outputs = stage_list[_source_id].engine_outputs
stage_list[_source_id].engine_outputs = source_outputs_override
try:
return self.custom_process_input_func(
stage_list, engine_input_source, prompt, self.requires_multimodal_data
)
finally:
stage_list[_source_id].engine_outputs = _orig_outputs
return self.custom_process_input_func(
stage_list, engine_input_source, prompt, self.requires_multimodal_data
)
@@ -702,6 +727,8 @@ def _stage_worker(
connectors_config = stage_payload.get("connectors_config", {})
stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm")
cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
if stage_type != "diffusion":
_resolve_worker_cls(engine_args)
@@ -761,6 +788,7 @@ def _stage_worker(
model=model,
stage_id=stage_id,
engine_input_source=stage_payload.get("engine_input_source", []),
cfg_kv_collect_func=cfg_kv_collect_func,
**engine_args,
)
else:
@@ -1109,6 +1137,7 @@ async def _stage_worker_async(
final_output = stage_payload.get("final_output", False)
final_output_type = stage_payload.get("final_output_type", None)
cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
# Handle non-standard model directory structures (e.g., tokenizer in root, model in subdir)
model = _resolve_model_tokenizer_paths(model, engine_args)
@@ -1187,10 +1216,13 @@ async def _stage_worker_async(
od_config["omni_kv_config"]["engine_input_source"] = stage_payload.get("engine_input_source", [])
logger.debug(f"[Stage-%s] Initializing diffusion engine with config: {od_config}", stage_id)
_diffusion_kwargs = {k: v for k, v in engine_args.items() if k not in {"od_config", "model"}}
if cfg_kv_collect_func is not None:
_diffusion_kwargs["cfg_kv_collect_func"] = cfg_kv_collect_func
stage_engine = AsyncOmniDiffusion(
model=model,
od_config=od_config,
**{k: v for k, v in engine_args.items() if k not in {"od_config", "model"}},
**_diffusion_kwargs,
)
vllm_config = None # Diffusion doesn't use vllm_config
else:
+1 -2
View File
@@ -13,7 +13,6 @@ import torch
from fastapi import Request
from PIL import Image
from pydantic import TypeAdapter
from vllm.renderers import BaseRenderer
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.protocol.chat_completion import OmniChatCompletionResponse
@@ -67,7 +66,7 @@ from vllm.inputs.data import PromptType
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from vllm.reasoning import ReasoningParser
from vllm.renderers import merge_kwargs
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs import TokPrompt
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
+18
View File
@@ -1,9 +1,11 @@
from __future__ import annotations
import enum
import importlib
import json
import logging
import os
from collections.abc import Callable
from multiprocessing import shared_memory as _shm
from typing import Any
@@ -12,6 +14,22 @@ from omegaconf import OmegaConf
logger = logging.getLogger(__name__)
def load_func_from_config(func_path: str | None) -> Callable[..., Any] | None:
"""Dynamically import a callable from a fully-qualified dotted path.
Args:
func_path: Dotted path such as ``"pkg.module.func_name"``, or *None*.
Returns:
The imported callable, or *None* when *func_path* is falsy.
"""
if not func_path:
return None
module_path, func_name = func_path.rsplit(".", 1)
module = importlib.import_module(module_path)
return getattr(module, func_name)
class OmniStageTaskType(enum.Enum):
GENERATE = "generate"
ABORT = "abort"