Merge branch 'main' into dev/vllm-align

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-01-30 02:53:03 +00:00
commit 4b895459e4
196 files changed
+9406 -4132

No files matched your search

+60 -93
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import copy
import time
import weakref
from collections.abc import AsyncGenerator, Iterable
from collections.abc import AsyncGenerator, Iterable, Sequence
from dataclasses import asdict
from pprint import pformat
from typing import Any
@@ -32,6 +33,7 @@ from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
from vllm_omni.entrypoints.utils import (
get_final_stage_id_for_e2e,
)
from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams
# Internal imports (our code)
from vllm_omni.lora.request import LoRARequest
@@ -66,10 +68,8 @@ class AsyncOmni(OmniBase):
asynchronous LLM and Diffusion models.
Args:
*args: Variable length argument list.
- args[0]: Model name or path to load.
model: Model name or path to load.
**kwargs: Arbitrary keyword arguments.
- model: Model name or path to load (if not in args).
- stage_configs_path: Optional path to YAML file containing stage
configurations. If None, configurations are loaded from the model.
- log_stats: Whether to enable statistics logging
@@ -95,7 +95,7 @@ class AsyncOmni(OmniBase):
... print(output)
"""
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
def __init__(self, model: str, **kwargs: dict[str, Any]) -> None:
# Pause/resume control attributes
self._pause_cond: asyncio.Condition = asyncio.Condition()
self._paused: bool = False
@@ -104,7 +104,7 @@ class AsyncOmni(OmniBase):
self.request_states: dict[str, ClientRequestState] = {}
self.output_handler: asyncio.Task | None = None
super().__init__(*args, **kwargs)
super().__init__(model, **kwargs)
# Register weak reference cleanup (called on garbage collection)
self._weak_finalizer = weakref.finalize(
@@ -163,6 +163,7 @@ class AsyncOmni(OmniBase):
"vae_use_tiling": kwargs.get("vae_use_tiling", False),
"cache_backend": cache_backend,
"cache_config": cache_config,
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
"enforce_eager": kwargs.get("enforce_eager", False),
},
@@ -233,7 +234,14 @@ class AsyncOmni(OmniBase):
if hasattr(self, "_weak_finalizer"):
self._weak_finalizer()
async def generate(self, *args: Any, **kwargs: dict[str, Any]) -> AsyncGenerator[OmniRequestOutput, None]:
async def generate(
self,
prompt: OmniPromptType,
request_id: str,
sampling_params_list: Sequence[OmniSamplingParams] | None = None,
*,
output_modalities: list[str] | None = None,
) -> AsyncGenerator[OmniRequestOutput, None]:
"""Generate outputs for the given prompt asynchronously.
Coordinates multi-stage pipeline through YAML configuration.
@@ -243,21 +251,13 @@ class AsyncOmni(OmniBase):
sampling parameters from the sampling_params_list.
Args:
*args: Arguments for generation.
- prompt: Prompt to process. Can be a text string, token IDs,
or multimodal prompt.
- request_id: Unique identifier for this request
- sampling_params_list: List of SamplingParams, one for each stage.
Must have the same length as the number of stages.
If None, uses default sampling params for each stage.
**kwargs: Additional arguments for generation.
- prompt: Prompt to process. Can be a text string, token IDs,
or multimodal prompt.
- request_id: Unique identifier for this request
- sampling_params_list: List of SamplingParams, one for each stage.
Must have the same length as the number of stages.
If None, uses default sampling params for each stage.
- output_modalities: Optional list of output modalities.
prompt: Prompt to process. Can be a text string, token IDs,
or multimodal prompt.
request_id: Unique identifier for this request
sampling_params_list: List of SamplingParams, one for each stage.
Must have the same length as the number of stages.
If None, uses default sampling params for each stage.
output_modalities: Optional list of output modalities.
Yields:
OmniRequestOutput objects as they are produced by each stage.
@@ -276,33 +276,9 @@ class AsyncOmni(OmniBase):
# Start output handler on the first call to generate()
self._run_output_handler()
prompt = args[0] if args else kwargs.get("prompt")
request_id = args[1] if len(args) > 1 else kwargs.get("request_id")
sampling_params_list = args[2] if len(args) > 2 else kwargs.get("sampling_params_list")
output_modalities = kwargs.get("output_modalities", None)
# TODO: lora_request, trace_headers, priority are not supported yet
if sampling_params_list is None:
# For Omni LLM, the params are parsed via the yaml file. For the current version,
# diffusion params can parsed via the command line.
omni_params_kwargs = {
k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"]
}
per_stage_params: list[Any] = []
for stage_id, stage in enumerate(self.stage_list):
stage_type = getattr(stage, "stage_type", "llm")
if stage_type == "diffusion":
default_dict = self.default_sampling_params_list[stage_id]
# Merge user-provided kwargs
merged = {**default_dict, **omni_params_kwargs}
# Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest
per_stage_params.append(merged)
else:
# LLM directly constructs SamplingParams, don't use the merged params
per_stage_params.append(self.default_sampling_params_list[stage_id])
sampling_params_list = per_stage_params
sampling_params_list = self.default_sampling_params_list
if len(sampling_params_list) != len(self.stage_list):
raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}")
@@ -329,38 +305,25 @@ class AsyncOmni(OmniBase):
req_state = ClientRequestState(request_id)
req_state.metrics = metrics
self.request_states[request_id] = req_state
sp0: SamplingParams = sampling_params_list[0] # type: ignore[index]
task = {
"request_id": request_id,
"engine_inputs": prompt,
"sampling_params": sp0,
}
self.stage_list[0].submit(task)
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
_req_start_ts[request_id] = time.time()
logger.info(
f"[{self._name}] Entering scheduling loop: stages={num_stages}, final_stage={final_stage_id_for_e2e}"
)
if self.async_chunk:
stage_queues = {stage_id: asyncio.Queue() for stage_id in range(num_stages)}
req_state.stage_queues = stage_queues
for i in range(num_stages):
sp: SamplingParams = sampling_params_list[i]
engine_inputs = prompt
if i != 0:
prompt_token_ids = prompt["prompt_token_ids"]
prompt_1 = prompt.copy()
prompt_1["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids)
prompt_1["multi_modal_data"] = prompt_1["mm_processor_kwargs"] = None
engine_inputs = prompt_1
task = {
"request_id": request_id,
"engine_inputs": engine_inputs,
"sampling_params": sp,
}
self.stage_list[i].submit(task)
metrics.stage_first_ts[i] = metrics.stage_first_ts[0] or time.time()
logger.info(f"[{self._name}] Enqueued request {request_id} to stage-{str(i)}")
_req_start_ts[request_id] = time.time()
logger.info(
f"[{self._name}] Entering scheduling loop: "
f"stages={num_stages}, final_stage={final_stage_id_for_e2e}"
)
async for output in self._process_async_results(
request_id,
prompt,
sampling_params_list,
req_state,
metrics,
final_stage_id_for_e2e,
@@ -369,22 +332,6 @@ class AsyncOmni(OmniBase):
):
yield output
else:
sp0: SamplingParams = sampling_params_list[0] # type: ignore[index]
task = {
"request_id": request_id,
"engine_inputs": prompt,
"sampling_params": sp0,
}
self.stage_list[0].submit(task)
_req_start_ts[request_id] = time.time()
# Mark first input time for stage-0
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
logger.info(
f"[{self._name}] Entering scheduling loop: "
f"stages={num_stages}, final_stage={final_stage_id_for_e2e}"
)
async for output in self._process_sequential_results(
request_id,
req_state,
@@ -415,6 +362,8 @@ class AsyncOmni(OmniBase):
async def _process_async_results(
self,
request_id: str,
prompt: Any,
sampling_params_list: list[SamplingParams],
req_state: ClientRequestState,
metrics: OrchestratorMetrics,
final_stage_id_for_e2e: int,
@@ -422,16 +371,34 @@ class AsyncOmni(OmniBase):
wall_start_ts: float,
) -> AsyncGenerator[OmniRequestOutput, None]:
all_stages_finished = {stage_id: False for stage_id in range(final_stage_id_for_e2e + 1)}
submit_flag = True
while not all(all_stages_finished.values()):
for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]):
if all_stages_finished[stage_id]:
continue
result = await req_state.stage_queues[stage_id].get()
logger.info(f"[{self._name}] Received result from stage-{stage_id}: {result}")
try:
result = req_state.stage_queues[stage_id].get_nowait()
except asyncio.QueueEmpty:
await asyncio.sleep(0.001)
continue
engine_outputs, finished, output_to_yield = self._process_single_result(
result, stage, stage_id, metrics, req_start_ts, wall_start_ts, final_stage_id_for_e2e
)
if submit_flag and stage_id == 0:
submit_flag = False
prompt_token_ids = engine_outputs.prompt_token_ids
engine_input = copy.deepcopy(prompt)
engine_input["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids)
engine_input["multi_modal_data"] = engine_input["mm_processor_kwargs"] = None
for i in range(1, len(self.stage_list)):
task = {
"request_id": request_id,
"engine_inputs": engine_input,
"sampling_params": sampling_params_list[i],
}
self.stage_list[i].submit(task)
metrics.stage_first_ts[i] = time.time()
all_stages_finished[stage_id] = finished
if output_to_yield:
+22 -98
View File
@@ -12,16 +12,15 @@ import asyncio
import uuid
from collections.abc import AsyncGenerator, Iterable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import fields
from typing import Any
from PIL import Image
from vllm.logger import init_logger
from vllm.transformers_utils.config import get_hf_file_to_dict
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
from vllm_omni.lora.request import LoRARequest
from vllm_omni.outputs import OmniRequestOutput
@@ -109,68 +108,19 @@ class AsyncOmniDiffusion:
logger.info("AsyncOmniDiffusion initialized with model: %s", model)
def _prepare_request(
self,
prompt: str,
request_id: str | None = None,
**kwargs: Any,
) -> OmniDiffusionRequest:
"""Prepare a diffusion request from prompt and parameters.
Args:
prompt: Text prompt for image generation
request_id: Optional unique identifier for the request
**kwargs: Additional generation parameters
Returns:
OmniDiffusionRequest ready for processing
"""
if request_id is None:
request_id = f"diff-{uuid.uuid4().hex[:16]}"
field_names = {f.name for f in fields(OmniDiffusionRequest)}
init_kwargs = {
"prompt": prompt,
"request_id": request_id,
}
for key, value in kwargs.items():
if key in field_names:
init_kwargs[key] = value
if "guidance_scale" in kwargs:
init_kwargs["guidance_scale_provided"] = True
return OmniDiffusionRequest(**init_kwargs)
async def generate(
self,
prompt: str,
prompt: OmniPromptType,
sampling_params: OmniDiffusionSamplingParams,
request_id: str | None = None,
num_inference_steps: int = 50,
guidance_scale: float | None = None,
height: int | None = None,
width: int | None = None,
negative_prompt: str | None = None,
num_outputs_per_prompt: int = 1,
seed: int | None = None,
lora_request=None,
**kwargs: Any,
lora_request: LoRARequest | None = None,
) -> OmniRequestOutput:
"""Generate images asynchronously from a text prompt.
Args:
prompt: Text prompt describing the desired image
sampling_params: Sampling parameters
request_id: Optional unique identifier for tracking the request
num_inference_steps: Number of denoising steps (default: 50)
guidance_scale: Classifier-free guidance scale (optional, uses model defaults if omitted)
height: Optional image height in pixels
width: Optional image width in pixels
negative_prompt: Optional negative prompt for guidance
num_outputs_per_prompt: Number of images to generate (default: 1)
seed: Optional random seed for reproducibility
**kwargs: Additional generation parameters
Returns:
OmniRequestOutput containing generated images
@@ -181,64 +131,38 @@ class AsyncOmniDiffusion:
if request_id is None:
request_id = f"diff-{uuid.uuid4().hex[:16]}"
# Prepare request
request_kwargs = {
"prompt": prompt,
"request_id": request_id,
"num_inference_steps": num_inference_steps,
"height": height,
"width": width,
"negative_prompt": negative_prompt,
"num_outputs_per_prompt": num_outputs_per_prompt,
"seed": seed,
"lora_request": lora_request,
**kwargs,
}
if guidance_scale is not None:
request_kwargs["guidance_scale"] = guidance_scale
if sampling_params.guidance_scale:
sampling_params.guidance_scale_provided = True
request = self._prepare_request(**request_kwargs)
if lora_request is not None:
sampling_params.lora_request = lora_request
request = OmniDiffusionRequest(
prompts=[prompt],
sampling_params=sampling_params,
request_ids=[request_id],
)
logger.debug("Starting generation for request %s", request_id)
# Run engine in thread pool
loop = asyncio.get_event_loop()
try:
# In async mode, only a single request is submitted at a time
result = await loop.run_in_executor(
self._executor,
self.engine.step,
[request],
request,
)
result = result[0]
except Exception as e:
logger.error("Generation failed for request %s: %s", request_id, e)
raise RuntimeError(f"Diffusion generation failed: {e}") from e
# Check if result is already OmniRequestOutput
if isinstance(result, OmniRequestOutput):
# Update request_id if needed
if not result.request_id:
result.request_id = request_id
return result
# Process results if not OmniRequestOutput
images: list[Image.Image] = []
if result is not None:
if isinstance(result, list):
for item in result:
if isinstance(item, Image.Image):
images.append(item)
elif isinstance(result, Image.Image):
images.append(result)
return OmniRequestOutput.from_diffusion(
request_id=request_id,
images=images,
prompt=prompt,
metrics={
"num_inference_steps": num_inference_steps,
"guidance_scale": request.guidance_scale,
},
)
# Update request_id if needed
if not result.request_id:
result.request_id = request_id
return result
async def generate_stream(
self,
+5
View File
@@ -175,6 +175,11 @@ class OmniServeCommand(CLISubcommand):
default=None,
help="JSON string of cache configuration (e.g., '{\"rel_l1_thresh\": 0.2}').",
)
omni_config_group.add_argument(
"--enable-cache-dit-summary",
action="store_true",
help="Enable cache-dit summary logging after diffusion forward passes.",
)
# VAE memory optimization parameters
omni_config_group.add_argument(
+66 -66
View File
@@ -10,14 +10,13 @@ from collections.abc import Callable, Generator, Sequence
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict
from pprint import pformat
from typing import Any
from typing import Any, Literal, overload
from omegaconf import OmegaConf
from tqdm.auto import tqdm
from vllm.inputs import PromptType
from vllm import SamplingParams
from vllm.logger import init_logger
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.distributed.omni_connectors import (
get_stage_connector_config,
initialize_orchestrator_connectors,
@@ -42,6 +41,7 @@ from vllm_omni.entrypoints.utils import (
load_stage_configs_from_yaml,
resolve_model_config_path,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
@@ -82,10 +82,8 @@ class OmniBase:
"""Base class for serving Omni models.
Args:
*args: Variable length argument list.
- args[0]: Model name or path to load.
model: Model name or path to load.
**kwargs: Arbitrary keyword arguments.
- model: Model name or path to load (if not in args).
- stage_configs_path: Optional path to YAML file containing stage
configurations. If None, configurations are loaded from the model.
- log_stats: Whether to enable statistics logging
@@ -102,14 +100,9 @@ class OmniBase:
- Additional keyword arguments passed to stage engines.
"""
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
model = args[0] if args else kwargs.get("model", "")
assert model != "", "Null model id detected, please specify a model id."
def __init__(self, model: str, **kwargs: Any) -> None:
model = omni_snapshot_download(model)
if args:
args[0] = model
elif kwargs.get("model", "") != "":
kwargs["model"] = model
kwargs["model"] = model
# Stage management attributes
self.stage_list: list[OmniStage] = []
@@ -502,10 +495,8 @@ class Omni(OmniBase):
"""Unified entrypoint for both LLM and Diffusion models for better usability.
Args:
*args: Variable length argument list.
- args[0]: Model name or path to load.
model: Model name or path to load.
**kwargs: Arbitrary keyword arguments.
- model: Model name or path to load (if not in args).
- stage_configs_path: Optional path to YAML file containing stage
configurations. If None, configurations are loaded from the model.
- log_stats: Whether to enable statistics logging
@@ -527,8 +518,8 @@ class Omni(OmniBase):
>>> print(outputs)
"""
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
super().__init__(*args, **kwargs)
def __init__(self, model: str, **kwargs: Any) -> None:
super().__init__(model, **kwargs)
# Register weak reference cleanup (called on garbage collection)
self._weak_finalizer = weakref.finalize(
@@ -539,8 +530,31 @@ class Omni(OmniBase):
self._ray_pg,
)
@overload
def generate(
self, *args: Any, **kwargs: dict[str, Any]
self,
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
*,
py_generator: Literal[True],
) -> Generator[OmniRequestOutput, None, None]: ...
@overload
def generate(
self,
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
*,
py_generator: Literal[False] = False,
) -> list[OmniRequestOutput]: ...
def generate(
self,
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
*,
py_generator: bool = False,
use_tqdm: bool | Callable[..., tqdm] = True,
) -> Generator[OmniRequestOutput, None, None] | list[OmniRequestOutput]:
"""Generate outputs for the given prompts.
@@ -548,12 +562,10 @@ class Omni(OmniBase):
Each stage will use OmniLLM or OmniDiffusion based on stage_type.
Args:
*args: Variable length argument list.
- args[0]: Input prompts for generation.
- args[1]: Optional list of per-stage parameters.
**kwargs: Arbitrary keyword arguments.
- prompt: Input prompts for generation (if not in args).
- sampling_params_list: Optional list of per-stage parameters (if not in args).
prompts: Input prompt(s) for generation.
sampling_params_list: Optional list of per-stage parameters.
py_generator: Whether the returned result(s) are wrapped in a generator instead of a list.
use_tqdm: Whether to use tqdm progress bar
Returns:
List of OmniRequestOutput objects, one for each input prompt.
@@ -563,40 +575,26 @@ class Omni(OmniBase):
Raises:
ValueError: If sampling_params_list is None or has incorrect length.
"""
prompts = args[0] if args else kwargs.get("prompts")
sampling_params_list = args[1] if len(args) > 1 else kwargs.get("sampling_params_list")
py_generator = kwargs.get("py_generator", False)
if prompts is None:
if kwargs.get("prompt") is None:
raise ValueError("prompts is required for generation")
prompts = kwargs.get("prompt")
if sampling_params_list is None:
# For Omni LLM, the params are parsed via the yaml file. For the current version,
# diffusion params can parsed via the command line.
omni_params_kwargs = {
k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"]
}
per_stage_params: list[Any] = []
for stage_id, stage in enumerate(self.stage_list):
stage_type = getattr(stage, "stage_type", "llm")
if stage_type == "diffusion":
default_dict = self.default_sampling_params_list[stage_id]
# Merge user-provided kwargs
merged = {**default_dict, **omni_params_kwargs}
# Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest
per_stage_params.append(merged)
sampling_params_list = self.default_sampling_params_list
elif not isinstance(sampling_params_list, Sequence):
# TODO: After the recent introduction of BAGEL model (one LLM and one Diffusion),
# expect the text_to_image example code to run when only passing one OmniDiffusionSamplingParams
# This behavior may be confusing, and future PR can improve it.
per_stage_params: list[OmniSamplingParams] = []
for default_stage_sp in self.default_sampling_params_list:
default_sp_type = default_stage_sp.__class__
if default_sp_type == sampling_params_list.__class__:
per_stage_params.append(sampling_params_list)
else:
# LLM directly constructs SamplingParams, don't use the merged params
per_stage_params.append(self.default_sampling_params_list[stage_id])
per_stage_params.append(default_stage_sp)
sampling_params_list = per_stage_params
try:
if py_generator:
return self._run_generation_with_generator(prompts, sampling_params_list)
else:
outputs = list(self._run_generation(prompts, sampling_params_list))
outputs = list(self._run_generation(prompts, sampling_params_list, use_tqdm))
return outputs
except Exception as e:
logger.exception("[Orchestrator] Failed to run generation: %s", e)
@@ -606,8 +604,8 @@ class Omni(OmniBase):
def _run_generation_with_generator(
self,
prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest],
sampling_params_list: Any | Sequence[Any] | None,
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params_list: Sequence[OmniSamplingParams],
) -> Generator[OmniRequestOutput, None, None]:
"""Run generation through all stages in the pipeline and return a generator."""
gen = self._run_generation(prompts, sampling_params_list)
@@ -622,8 +620,8 @@ class Omni(OmniBase):
def _run_generation(
self,
prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest],
sampling_params_list: Any | Sequence[Any] | None = None,
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params_list: Sequence[OmniSamplingParams],
use_tqdm: bool | Callable[..., tqdm] = True,
) -> Generator[OmniRequestOutput, None, None]:
"""Run generation through all stages in the pipeline."""
@@ -631,18 +629,20 @@ class Omni(OmniBase):
if sampling_params_list is None:
raise ValueError("sampling_params_list is required for pipelined generation")
# Normalize sampling_params_list to a list
if not isinstance(sampling_params_list, (list, tuple)):
sampling_params_list = [sampling_params_list]
else:
sampling_params_list = list(sampling_params_list)
if len(sampling_params_list) != len(self.stage_list):
raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}")
for i, (stage, sp) in enumerate(zip(self.stage_list, sampling_params_list)):
ExpectedSPType = OmniDiffusionSamplingParams if stage.stage_type == "diffusion" else SamplingParams
if not isinstance(sp, ExpectedSPType):
raise ValueError(
f"Expected sampling parameters with type {ExpectedSPType} in stage {i}, got {sp.__class__}"
)
# Normalize prompts to a list for per-request iteration
if not isinstance(prompts, (list, tuple)):
request_prompts: list[PromptType] = [prompts]
# str is also Sequence but only test list-like containers here
if isinstance(prompts, str) or not isinstance(prompts, Sequence):
request_prompts: list[OmniPromptType] = [prompts]
else:
request_prompts = list(prompts)
@@ -650,8 +650,8 @@ class Omni(OmniBase):
num_stages = len(self.stage_list)
# Generate globally unique request IDs and map them to original prompts
request_ids: list[str] = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))]
request_id_to_prompt: dict[str, PromptType] = {rid: p for rid, p in zip(request_ids, request_prompts)}
request_ids = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))]
request_id_to_prompt = {rid: p for rid, p in zip(request_ids, request_prompts)}
# Track per-request start time for end-to-end timing
_req_start_ts: dict[str, float] = {}
+16 -47
View File
@@ -3,7 +3,7 @@
import logging
import uuid
from dataclasses import fields
from collections.abc import Sequence
from vllm.logger import init_logger
from vllm.transformers_utils.config import get_hf_file_to_dict
@@ -11,6 +11,8 @@ from vllm.transformers_utils.config import get_hf_file_to_dict
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
from vllm_omni.outputs import OmniRequestOutput
# TODO configure logging properly
logging.basicConfig(level=logging.INFO)
@@ -18,21 +20,6 @@ logging.basicConfig(level=logging.INFO)
logger = init_logger(__name__)
def prepare_requests(prompt: str | list[str], **kwargs):
field_names = {f.name for f in fields(OmniDiffusionRequest)}
init_kwargs = {"prompt": prompt}
for key, value in kwargs.items():
if key in field_names:
init_kwargs[key] = value
if "guidance_scale" in kwargs:
init_kwargs["guidance_scale_provided"] = True
return OmniDiffusionRequest(**init_kwargs)
class OmniDiffusion:
"""
It is the main class to interact with vLLM-Omni diffusion models.
@@ -100,42 +87,24 @@ class OmniDiffusion:
def generate(
self,
prompt: str | list[str],
**kwargs,
):
prompts = []
if isinstance(prompt, str):
prompts.append(prompt)
elif isinstance(prompt, list):
prompts.extend(prompt)
prompts: OmniPromptType | Sequence[OmniPromptType],
sampling_params: OmniDiffusionSamplingParams,
request_ids: list[str] = [],
) -> list[OmniRequestOutput]:
if isinstance(prompts, (str, dict)):
prompts = [prompts]
else:
raise ValueError("Prompt must be a string or a list of strings")
requests: list[OmniDiffusionRequest] = []
prompts = list(prompts)
# Check if request_id is provided in kwargs
request_id = kwargs.get("request_id")
request_ids = kwargs.pop("request_ids", None)
if len(request_ids) < len(prompts):
request_ids.extend(f"{i + len(request_ids)}_{uuid.uuid4()}" for i in range(len(prompts) - len(request_ids)))
for i, p in enumerate(prompts):
req_kwargs = kwargs.copy()
if request_ids and isinstance(request_ids, list) and i < len(request_ids):
req_kwargs["request_id"] = request_ids[i]
elif request_id is None:
# Generate default ID consistent with OmniLLM: "{i}_{uuid}"
req_kwargs["request_id"] = f"{i}_{uuid.uuid4()}"
request = OmniDiffusionRequest(prompts, sampling_params, request_ids)
return self._run_engine(request)
requests.append(
prepare_requests(
p,
**req_kwargs,
)
)
logger.info(f"Prepared {len(requests)} requests for generation.")
return self._run_engine(requests)
def _run_engine(self, requests: list[OmniDiffusionRequest]):
return self.engine.step(requests)
def _run_engine(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]:
return self.engine.step(request)
def close(self) -> None:
self.engine.close()
+382 -354
View File
@@ -15,9 +15,11 @@ import queue
import sys
import time
import traceback
from collections.abc import Sequence
from dataclasses import fields
from typing import Any
from typing import Any, Literal, cast
from vllm import PromptType, RequestOutput
from vllm.inputs import TextPrompt
from vllm.inputs.preprocess import InputPreprocessor
from vllm.logger import init_logger
@@ -31,6 +33,7 @@ from vllm.v1.engine.llm_engine import LLMEngine
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.distributed.omni_connectors import build_stage_connectors
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector
from vllm_omni.distributed.omni_connectors.connectors.base import OmniConnectorBase
from vllm_omni.distributed.ray_utils.utils import kill_ray_actor, start_ray_actor
from vllm_omni.engine.arg_utils import AsyncOmniEngineArgs
from vllm_omni.entrypoints.async_omni_diffusion import AsyncOmniDiffusion
@@ -46,12 +49,29 @@ from vllm_omni.entrypoints.stage_utils import (
maybe_dump_to_shm,
set_stage_devices,
)
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.utils import detect_device_type
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams, OmniTokensPrompt
from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
def _resolve_worker_cls(engine_args: dict[str, Any]) -> None:
worker_type = engine_args.pop("worker_type", None)
if not worker_type:
return
if engine_args.get("worker_cls"):
return
from vllm_omni.platforms import current_omni_platform
worker_type = str(worker_type).lower()
if worker_type == "ar":
engine_args["worker_cls"] = current_omni_platform.get_omni_ar_worker_cls()
elif worker_type == "generation":
engine_args["worker_cls"] = current_omni_platform.get_omni_generation_worker_cls()
else:
raise ValueError(f"Unknown worker_type: {worker_type}")
def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]:
"""Build OmniDiffusionConfig kwargs from engine args."""
od_config = engine_args.get("od_config", {})
@@ -64,35 +84,6 @@ def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]:
return od_config
def prepare_sampling_params(sampling_params: Any, stage_type: str) -> Any:
"""Prepare sampling parameters for the given stage type.
Args:
sampling_params: Raw sampling parameters (dict or SamplingParams)
stage_type: Either "llm" or "diffusion"
Returns:
Processed sampling parameters ready for engine consumption
"""
if stage_type == "diffusion":
# For diffusion stages: extract kwargs, handling different input types
if isinstance(sampling_params, dict):
diffusion_kwargs = dict(sampling_params)
else:
diffusion_kwargs = getattr(sampling_params, "__dict__", {}) or {}
# Remove 'prompt' and 'request_id' to avoid conflict with explicit arguments
diffusion_kwargs.pop("prompt", None)
diffusion_kwargs.pop("request_id", None)
return diffusion_kwargs
else: # stage_type == "llm"
# For LLM stages: ensure we have a SamplingParams object
if isinstance(sampling_params, dict):
return SamplingParams(**sampling_params)
return sampling_params
class OmniStage:
"""Stage manager for orchestrating a single stage in the omni pipeline.
@@ -123,7 +114,7 @@ class OmniStage:
self.engine_outputs = None
self.is_comprehension = getattr(stage_config, "is_comprehension", False)
# Support for different stage types: "llm" (default) or "diffusion"
self.stage_type = getattr(stage_config, "stage_type", "llm")
self.stage_type: Literal["llm", "diffusion"] = getattr(stage_config, "stage_type", "llm")
if hasattr(stage_config, "custom_process_input_func"):
# Import the module specified in the config (already a full module path)
module_path, func_name = stage_config.custom_process_input_func.rsplit(".", 1)
@@ -137,7 +128,14 @@ class OmniStage:
default_sampling_params = getattr(stage_config, "default_sampling_params", {})
# For LLM stage, this can directly be a SamplingParams-compatible dict;
# For diffusion stage, this only serves as default values for diffusion kwargs.
self.default_sampling_params = _to_dict(default_sampling_params)
default_sampling_params = _to_dict(default_sampling_params)
# Further convert it to dataclass to check fields
try:
self.default_sampling_params = (
SamplingParams if self.stage_type == "llm" else OmniDiffusionSamplingParams
)(**default_sampling_params)
except TypeError as error:
raise TypeError(f"Invalid default_sampling_params for stage {self.stage_id}: {error}") from error
# Runtime orchestration state (added)
self._in_q: mp.Queue | None = None
self._out_q: mp.Queue | None = None
@@ -501,6 +499,9 @@ def _stage_worker(
import os as _os
import time as _time
from vllm_omni.plugins import load_omni_general_plugins
load_omni_general_plugins()
# IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker /
# GPUARModelRunner) are spawned with a fork-safe method.
# Mooncake / gRPC / RDMA and CUDA/NCCL can deadlock under fork-with-threads.
@@ -519,7 +520,10 @@ def _stage_worker(
runtime_cfg = stage_payload.get("runtime", {})
shm_threshold_bytes = int(stage_payload.get("shm_threshold_bytes", 65536))
connectors_config = stage_payload.get("connectors_config", {})
stage_type = stage_payload.get("stage_type", "llm")
stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm")
if stage_type != "diffusion":
_resolve_worker_cls(engine_args)
# Aggregates for running average
_agg_total_tokens = 0
@@ -530,7 +534,9 @@ def _stage_worker(
# Device mapping
device_type = None
try:
device_type = detect_device_type()
from vllm_omni.platforms import current_omni_platform
device_type = current_omni_platform.device_type
set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type)
except Exception as e:
logger.warning("Device setup failed: %s", e)
@@ -539,139 +545,137 @@ def _stage_worker(
# when multiple instances start simultaneously
# For TP/PP/DP/SP, we need to lock ALL devices that will be used by this stage
lock_files = []
if device_type == "cuda":
try:
import torch
try:
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1 # not use in omni model
cfg_parallel_size = 1 # not used in omni model
if torch.cuda.is_available():
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = 1 # not used for diffusion
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1 # not use in omni model
cfg_parallel_size = 1 # not used in omni model
# Calculate total number of devices needed for this stage
# For a single stage worker:
# - TP: splits model across devices (always needed)
# - PP: splits layers across pipeline stages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# - SP: sequence parallelism, typically uses TP devices
# - CFG: Classifier-Free Guidance parallelism for diffusion models
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Calculate total number of devices needed for this stage
# For a single stage worker:
# - TP: splits model across GPUs (always needed)
# - PP: splits layers across pipelinestages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# - SP: sequence parallelism, typically uses TP devices
# - CFG: Classifier-Free Guidance parallelism for diffusion models
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
# After set_stage_devices, this env var is set to physical device(s)
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = _os.environ.get(device_control_env)
physical_devices = []
# Get physical device IDs from CUDA_VISIBLE_DEVICES
# After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s)
cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES")
physical_devices = []
if visible_devices_str:
try:
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
except (ValueError, IndexError):
pass
if cuda_visible_devices:
if not physical_devices:
# Fallback: use logical device count if device control env var not set
num_devices = current_omni_platform.get_device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
cfg_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()]
except (ValueError, IndexError):
pass
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
if not physical_devices:
# Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set
num_devices = torch.cuda.device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway",
device_id,
)
break
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway",
device_id,
e,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
except Exception as e:
logger.debug(
"[Stage-%s] Failed to set up sequential initialization lock: %s",
stage_id,
e,
)
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
device_id,
e,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
except Exception as e:
logger.debug(
"[Stage-%s] Failed to set up sequential initialization lock: %s",
stage_id,
e,
)
# Init engine based on stage_type
logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys()))
if engine_args.get("async_chunk", False):
@@ -708,15 +712,14 @@ def _stage_worker(
pass
logger.debug("Engine initialized")
# Initialize OmniConnectors if configured
connectors = {}
connectors: dict[tuple[str, str], OmniConnectorBase] | None = {}
if connectors_config:
built_connectors = build_stage_connectors(
connectors = build_stage_connectors(
stage_id=stage_id,
connectors_config=connectors_config,
)
if built_connectors is None:
if connectors is None:
return
connectors = built_connectors
# Signal readiness to orchestrator
try:
@@ -785,6 +788,7 @@ def _stage_worker(
continue
batch_tasks: list[dict[str, Any]] = [task]
tasks_failed_to_add_to_batch: list[dict[str, Any]] = []
start_time = _time.time()
if max_batch_size > 1:
while len(batch_tasks) < max_batch_size:
@@ -800,7 +804,20 @@ def _stage_worker(
if extra_type == OmniStageTaskType.PROFILER_STOP:
out_q.put({"type": "profiler_result", "data": p_data})
continue
batch_tasks.append(extra)
# Ensure that all tasks have the same sampling params
# If no, put them in a temporary container and add back to queue
# This should be always true, because user only calls omni.generate() once and it blocks
# User can only pass one sampling param object, but the list of prompts are separated.
if task.get("sampling_params") != extra.get("sampling_params"):
logger.warning(
"""In offline mode, expect all prompts in one `omni.generate()` call to share same sampling params""" # noqa: E501 # line too long
f"""However, prompt {task.get("engine_inputs")} has sampling params {task.get("sampling_params")}, """ # noqa: E501 # line too long
f"""whereas the prompt {extra.get("engine_inputs")} has sampling params {extra.get("sampling_params")}.""" # noqa: E501 # line too long
"""The two tasks cannot be combined in one batch request."""
)
tasks_failed_to_add_to_batch.append(extra)
else:
batch_tasks.append(extra)
end_time = _time.time()
duration = end_time - start_time
if duration > batch_timeout:
@@ -815,9 +832,13 @@ def _stage_worker(
break
else:
continue
for task_to_readd in tasks_failed_to_add_to_batch:
in_q.put(task_to_readd)
# Ensure that the popped tasks are with identical sampling params. Take one of them.
batch_engine_sampling_params: OmniSamplingParams = batch_tasks[0]["sampling_params"]
batch_request_ids: list[Any] = []
batch_engine_inputs: list[Any] = []
batch_engine_inputs: list[OmniPromptType] = []
_rx_bytes_by_rid: dict[Any, int] = {}
_rx_decode_ms_by_rid: dict[Any, float] = {}
_in_flight_ms_by_rid: dict[Any, float] = {}
@@ -839,6 +860,9 @@ def _stage_worker(
connectors=connectors,
stage_id=stage_id,
)
# TODO: hack type annotation for now.
# A better way is to refine type annotation of connection and task/payloads, maybe using template types.
ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein)
if ein is None or _rx_metrics is None:
raise RuntimeError(
@@ -850,17 +874,14 @@ def _stage_worker(
_rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
batch_request_ids.append(rid)
if isinstance(ein, list):
if isinstance(ein, (str, dict)):
# Types like OmniTextPrompt, TextPrompt are TypedDict, essentially dict and enters this branch
batch_engine_inputs.append(ein)
elif isinstance(ein, Sequence):
batch_engine_inputs.extend(ein)
elif isinstance(ein, dict):
batch_engine_inputs.append(ein)
elif isinstance(ein, str):
# For diffusion stage-0, ein might be a string prompt directly
batch_engine_inputs.append(ein)
else:
# For other types (e.g., OmniTokensPrompt, TextPrompt), append as-is
# Other unknown types, append as-is
batch_engine_inputs.append(ein)
sampling_params = batch_tasks[0]["sampling_params"]
logger.debug(
"Received batch size=%d, request_ids=%s",
len(batch_tasks),
@@ -868,65 +889,30 @@ def _stage_worker(
)
try:
_batch_seq += 1
gen_outputs: list[Any] = []
gen_outputs: list[OmniRequestOutput | RequestOutput] = []
_gen_t0 = _time.time()
if stage_type == "diffusion":
# For diffusion, batch_engine_inputs should be prompts (strings)
# Convert to list of strings if needed
prompts = []
for ein in batch_engine_inputs:
if isinstance(ein, str):
prompts.append(ein)
elif isinstance(ein, dict) and "prompt" in ein:
prompts.append(ein["prompt"])
elif hasattr(ein, "prompt"):
prompts.append(ein.prompt)
else:
prompts.append(str(ein))
# Prepare diffusion kwargs from sampling parameters
diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion")
# Pass batch_request_ids to ensure correct ID mapping
diffusion_kwargs["request_ids"] = batch_request_ids
stage_engine = cast(OmniDiffusion, stage_engine)
batch_engine_sampling_params = cast(OmniDiffusionSamplingParams, batch_engine_sampling_params)
# Diffusion generate returns results directly, not an iterator
diffusion_results = stage_engine.generate(prompts, **diffusion_kwargs)
# Convert to list format compatible with LLM outputs
# Ensure each result has a request_id for proper mapping
if isinstance(diffusion_results, list):
gen_outputs = diffusion_results
# Assign request_ids if not present
for idx, result in enumerate(gen_outputs):
if not hasattr(result, "request_id") or result.request_id is None:
if idx < len(batch_request_ids):
if hasattr(result, "request_id"):
result.request_id = batch_request_ids[idx]
else:
# Create a wrapper object if result doesn't support request_id
from types import SimpleNamespace
wrapped = SimpleNamespace()
wrapped.request_id = batch_request_ids[idx]
wrapped.output = result
gen_outputs[idx] = wrapped
else:
gen_outputs = [diffusion_results]
# Assign request_id to single result
if len(batch_request_ids) > 0:
if hasattr(gen_outputs[0], "request_id"):
gen_outputs[0].request_id = batch_request_ids[0]
else:
from types import SimpleNamespace
wrapped = SimpleNamespace()
wrapped.request_id = batch_request_ids[0]
wrapped.output = gen_outputs[0]
gen_outputs[0] = wrapped
diffusion_results = stage_engine.generate(
batch_engine_inputs, batch_engine_sampling_params, batch_request_ids
)
gen_outputs.extend(diffusion_results)
# Assign request_ids if not present
for idx, result in enumerate(gen_outputs):
if not hasattr(result, "request_id") or result.request_id is None:
if idx < len(batch_request_ids):
result.request_id = batch_request_ids[idx]
else:
# LLM engine: use vLLM native SamplingParams
llm_sampling_params = prepare_sampling_params(sampling_params, "llm")
for ro in stage_engine.generate(batch_engine_inputs, llm_sampling_params, use_tqdm=False):
gen_outputs.append(ro)
stage_engine = cast(OmniLLM, stage_engine)
batch_engine_sampling_params = cast(SamplingParams, batch_engine_sampling_params)
results = stage_engine.generate(
batch_engine_inputs, # type: ignore # silent complaints about list of subclassed TypedDict
batch_engine_sampling_params,
use_tqdm=False,
)
gen_outputs.extend(results)
_gen_t1 = _time.time()
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
logger.debug(f"Generate done: batch={len(batch_tasks)}, req_ids={batch_request_ids}, gen_ms={_gen_ms:.1f}")
@@ -935,7 +921,7 @@ def _stage_worker(
req_to_outputs: dict[Any, list[Any]] = {rid: [] for rid in batch_request_ids}
unmapped: list[Any] = []
for ro in gen_outputs:
rid = getattr(ro, "request_id", None)
rid = ro.request_id
if rid in req_to_outputs:
req_to_outputs[rid].append(ro)
else:
@@ -1037,6 +1023,9 @@ async def _stage_worker_async(
import os as _os
import time as _time
from vllm_omni.plugins import load_omni_general_plugins
load_omni_general_plugins()
# IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker /
# GPUARModelRunner) are spawned with a fork-safe method.
if _os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn":
@@ -1054,6 +1043,9 @@ async def _stage_worker_async(
connectors_config = stage_payload.get("connectors_config", {})
stage_type = stage_payload.get("stage_type", "llm")
if stage_type != "diffusion":
_resolve_worker_cls(engine_args)
in_q = omni_stage._in_q
out_q = omni_stage._out_q
@@ -1067,9 +1059,9 @@ async def _stage_worker_async(
# Device mapping
device_type = None
try:
from vllm_omni.utils import detect_device_type
from vllm_omni.platforms import current_omni_platform
device_type = detect_device_type()
device_type = current_omni_platform.device_type
set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type)
except Exception as e:
logger.warning("Device setup failed: %s", e)
@@ -1087,120 +1079,136 @@ async def _stage_worker_async(
# Sequential initialization on the same device to avoid memory calculation errors
# when multiple instances start simultaneously
# For TP, we need to lock ALL devices that will be used by this stage
# For TP/PP/DP/PCP, we need to lock ALL devices that will be used by this stage
lock_files = []
if device_type == "cuda":
try:
import torch
try:
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1 # not use in omni model
cfg_parallel_size = 1 # not used in omni model
if torch.cuda.is_available():
# Get all parallel sizes from engine_args (defaults to 1)
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
# Calculate total number of devices needed for this stage
# For a single stage worker:
# - TP: splits model across devices (always needed)
# - PP: splits layers across pipeline stages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# - SP: sequence parallelism, typically uses TP devices
# - CFG: Classifier-Free Guidance parallelism for diffusion models
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Calculate total number of devices needed for this stage
# For a single stage worker in omni:
# - TP: splits model across GPUs (always needed)
# - PP: splits layers across stages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# The number of devices per stage is determined by TP * PP * DP * PCP size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size * pipeline_parallel_size * data_parallel_size * prefill_context_parallel_size
)
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
# After set_stage_devices, this env var is set to physical device(s)
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = _os.environ.get(device_control_env)
physical_devices = []
# Get physical device IDs from CUDA_VISIBLE_DEVICES
# After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s)
cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES")
physical_devices = []
if visible_devices_str:
try:
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
except (ValueError, IndexError):
pass
if cuda_visible_devices:
if not physical_devices:
# Fallback: use logical device count if device control env var not set
num_devices = current_omni_platform.get_device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
cfg_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()]
except (ValueError, IndexError):
pass
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
if not physical_devices:
# Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set
num_devices = torch.cuda.device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, "
"proceeding anyway with timeout %s",
device_id,
stage_init_timeout,
)
break
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway with timeout %s",
device_id,
e,
stage_init_timeout,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
except Exception as e:
logger.debug("Failed to set up sequential initialization lock: %s", e)
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
device_id,
e,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
except Exception as e:
logger.debug("Failed to set up sequential initialization lock: %s", e)
# Init engine based on stage_type
logger.debug(
@@ -1354,6 +1362,10 @@ async def _stage_worker_async(
connectors=connectors,
stage_id=stage_id,
)
# TODO: hack type annotation for now.
# A better way is to refine type annotation of connection and task/payloads, maybe using template types.
ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein)
if ein is None or _rx_metrics is None:
raise RuntimeError(
f"[Stage-{stage_id}] Missing connector payload for request {rid}. "
@@ -1362,36 +1374,23 @@ async def _stage_worker_async(
_rx_decode_ms_by_rid[rid] = float(_rx_metrics.get("rx_decode_time_ms", 0.0))
_rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
sampling_params = task["sampling_params"]
logger.debug("Received batch size=1, request_ids=%s", rid)
_gen_t0 = _time.time()
if isinstance(ein, list):
if isinstance(ein, Sequence) and not isinstance(ein, str):
ein = ein[0]
if stage_type == "diffusion":
# For diffusion, ein should be prompts (strings)
# Convert to string if needed
if isinstance(ein, str):
prompt = ein
elif isinstance(ein, dict) and "prompt" in ein:
prompt = ein["prompt"]
elif hasattr(ein, "prompt"):
prompt = ein.prompt
else:
prompt = str(ein)
# Prepare diffusion kwargs from sampling parameters
diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion")
diffusion_sampling_params = cast(OmniDiffusionSamplingParams, task["sampling_params"])
# AsyncOmniDiffusion.generate returns a single result, not an async generator
gen_output = await stage_engine.generate(prompt=prompt, request_id=rid, **diffusion_kwargs)
gen_output = await cast(AsyncOmniDiffusion, stage_engine).generate(ein, diffusion_sampling_params, rid)
_gen_t1 = _time.time()
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
await generation_out_q.put((rid, gen_output, _gen_ms))
else:
# LLM stages: ensure using SamplingParams
llm_sampling_params = prepare_sampling_params(sampling_params, "llm")
ein = cast(PromptType, ein)
llm_sampling_params: SamplingParams = task["sampling_params"]
gen_output = None
async for res in stage_engine.generate(ein, llm_sampling_params, rid):
async for res in cast(AsyncLLM, stage_engine).generate(ein, llm_sampling_params, rid):
gen_output = res
_gen_t1 = _time.time()
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
@@ -1468,7 +1467,7 @@ async def _stage_worker_async(
batch_request_ids, batch_request_outputs, _gen_ms_list, batch_metrics
):
try:
r_outputs = [output]
r_outputs = [output_strip(output, omni_stage)]
use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes)
if use_shm:
out_q.put(
@@ -1554,3 +1553,32 @@ def make_stage_stats(_agg_total_tokens: int, _agg_total_gen_time_ms: float):
from vllm_omni.entrypoints.log_utils import StageStats
return StageStats(total_token=_agg_total_tokens, total_gen_time=_agg_total_gen_time_ms)
def output_strip(r_output: RequestOutput | OmniRequestOutput, omni_stage: OmniStage):
"""
Strip unnecessary multimodal outputs from stages results,
in order to:
- reduce memory usage
- reduce transfer & serialization overhead
"""
# check multimodal data is required by stage output config.
if omni_stage.final_output and omni_stage.final_output_type != "text":
return r_output
# If the request has already finished, should not be altered.
if getattr(r_output, "finished", False):
return r_output
mm_output = getattr(r_output, "multimodal_output", None)
if mm_output is not None:
r_output.multimodal_output = {}
outputs = getattr(r_output, "outputs", None)
if outputs is not None:
for out in outputs:
if getattr(out, "multimodal_output", None):
out.multimodal_output = {}
return r_output
+142 -24
View File
@@ -5,18 +5,20 @@ import multiprocessing.forkserver as forkserver
import os
# Image generation API imports
import random
import time
from argparse import Namespace
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from http import HTTPStatus
from typing import Any
from typing import Any, cast
import vllm.envs as envs
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from starlette.datastructures import State
from starlette.routing import Route
from vllm import SamplingParams
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
@@ -79,12 +81,30 @@ from vllm_omni.entrypoints.openai.protocol.images import (
)
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
logger = init_logger(__name__)
router = APIRouter()
def _remove_route_from_router(
router: APIRouter,
path: str,
methods: set[str] | None = None,
) -> None:
methods_set = {method.upper() for method in methods} if methods else None
for route in list(router.routes):
if getattr(route, "path", None) != path:
continue
if methods_set is not None:
route_methods = {method.upper() for method in (getattr(route, "methods", None) or set())}
if not (route_methods & methods_set):
continue
router.routes.remove(route)
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
@@ -741,6 +761,9 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
return StreamingResponse(content=generator, media_type="text/event-stream")
_remove_route_from_router(router, "/v1/audio/speech", {"POST"})
@router.post(
"/v1/audio/speech",
dependencies=[Depends(validate_json_request)],
@@ -769,6 +792,86 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
# Health and Model endpoints for diffusion mode
# Remove existing health endpoint if present (from vllm imports)
# to ensure our handler takes precedence
_remove_route_from_router(router, "/health")
@router.get("/health")
async def health(raw_request: Request) -> JSONResponse:
"""Health check endpoint that works for both LLM and diffusion modes.
Returns 200 OK if the server is healthy.
For LLM mode: delegates to engine_client health check
For diffusion mode: checks if diffusion_engine is running
"""
# Check if we're in diffusion mode
diffusion_engine = getattr(raw_request.app.state, "diffusion_engine", None)
if diffusion_engine is not None:
# Diffusion mode health check
if hasattr(diffusion_engine, "is_running") and diffusion_engine.is_running:
return JSONResponse(content={"status": "healthy"})
return JSONResponse(
content={"status": "unhealthy", "reason": "Diffusion engine is not running"},
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)
# LLM mode - delegate to engine_client
engine_client = getattr(raw_request.app.state, "engine_client", None)
if engine_client is not None:
await engine_client.check_health()
return JSONResponse(content={"status": "healthy"})
return JSONResponse(
content={"status": "unhealthy", "reason": "No engine initialized"},
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)
# Remove existing models endpoint if present (from vllm imports)
# to ensure our handler takes precedence
_remove_route_from_router(router, "/v1/models")
@router.get("/v1/models")
async def show_available_models(raw_request: Request) -> JSONResponse:
"""Show available models endpoint that works for both LLM and diffusion modes.
Returns model information in OpenAI-compatible format.
"""
# Check if we're in diffusion mode
diffusion_model_name = getattr(raw_request.app.state, "diffusion_model_name", None)
if diffusion_model_name is not None:
# Diffusion mode - return the loaded model
return JSONResponse(
content={
"object": "list",
"data": [
{
"id": diffusion_model_name,
"object": "model",
"created": 0,
"owned_by": "vllm-omni",
"permission": [],
}
],
}
)
# LLM mode - delegate to openai_serving_models
openai_serving_models = getattr(raw_request.app.state, "openai_serving_models", None)
if openai_serving_models is not None:
models = await openai_serving_models.show_available_models()
return JSONResponse(content=models.model_dump())
return JSONResponse(
content={"object": "list", "data": []},
)
# Image generation API endpoints
@@ -799,7 +902,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
HTTPException: For validation errors, missing engine, or generation failures
"""
# Get engine client (AsyncOmni) from app state
engine_client: EngineClient | None = getattr(raw_request.app.state, "engine_client", None)
engine_client: EngineClient | AsyncOmni | None = getattr(raw_request.app.state, "engine_client", None)
if engine_client is None or not hasattr(engine_client, "stage_list"):
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
@@ -859,10 +962,8 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
try:
# Build params - pass through user values directly
gen_params = {
"prompt": request.prompt,
"num_outputs_per_prompt": request.n,
}
prompt: OmniTextPrompt = {"prompt": request.prompt}
gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n)
# Parse per-request LoRA (compatible with chat's extra_body.lora shape).
if request.lora is not None:
@@ -894,62 +995,79 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
detail="Invalid lora object: both name and path are required.",
)
gen_params["lora_request"] = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
gen_params.lora_request = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
if lora_scale is not None:
gen_params["lora_scale"] = float(lora_scale)
gen_params.lora_scale = float(lora_scale)
# Parse and add size if provided
if request.size:
width, height = parse_size(request.size)
gen_params["height"] = height
gen_params["width"] = width
gen_params.height = height
gen_params.width = width
size_str = f"{width}x{height}"
else:
size_str = "model default"
# Add optional parameters ONLY if provided
if request.num_inference_steps is not None:
gen_params["num_inference_steps"] = request.num_inference_steps
gen_params.num_inference_steps = request.num_inference_steps
if request.negative_prompt is not None:
gen_params["negative_prompt"] = request.negative_prompt
prompt["negative_prompt"] = request.negative_prompt
if request.guidance_scale is not None:
gen_params["guidance_scale"] = request.guidance_scale
gen_params.guidance_scale = request.guidance_scale
if request.true_cfg_scale is not None:
gen_params["true_cfg_scale"] = request.true_cfg_scale
gen_params.true_cfg_scale = request.true_cfg_scale
if request.seed is not None:
gen_params["seed"] = request.seed
gen_params["request_id"] = f"img_gen_{int(time.time())}"
gen_params.seed = request.seed
else:
# If seed is not provided, generate a random one to ensure
# a proper generator is initialized in the backend.
# This fixes issues where using the default global generator
# might produce blurry images in some environments.
gen_params.seed = random.randint(0, 2**32 - 1)
request_id = f"img_gen_{int(time.time())}"
logger.info(f"Generating {request.n} image(s) {size_str}")
# Generate images using AsyncOmni (multi-stage mode)
engine_client = cast(AsyncOmni, engine_client)
result = None
stage_list = getattr(engine_client, "stage_list", None)
if isinstance(stage_list, list):
default_params_list = getattr(engine_client, "default_sampling_params_list", None)
default_params_list: list[OmniSamplingParams] | None = getattr(
engine_client, "default_sampling_params_list", None
)
if not isinstance(default_params_list, list):
default_params_list = [{} for _ in stage_types]
default_params_list = [
OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types
]
else:
default_params_list = list(default_params_list)
if len(default_params_list) != len(stage_types):
default_params_list = (default_params_list + [{} for _ in stage_types])[: len(stage_types)]
default_params_list = (
default_params_list
+ [OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types]
)[: len(stage_types)]
sampling_params_list: list[dict[str, Any]] = []
sampling_params_list: list[OmniSamplingParams] = []
for idx, stage_type in enumerate(stage_types):
if stage_type == "diffusion":
sampling_params_list.append(gen_params)
else:
base_params = default_params_list[idx]
sampling_params_list.append(dict(base_params) if isinstance(base_params, dict) else base_params)
sampling_params_list.append(base_params)
async for output in engine_client.generate(
prompt=gen_params["prompt"],
request_id=gen_params["request_id"],
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
):
result = output
else:
result = await engine_client.generate(**gen_params)
result = await engine_client.generate(
prompt=prompt, request_id=request_id, sampling_params_list=[gen_params]
)
if result is None:
raise HTTPException(
+39 -28
View File
@@ -6,13 +6,17 @@ import uuid
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
from datetime import datetime, timedelta, timezone
from io import BytesIO
from typing import TYPE_CHECKING, Any, Final, Optional
from typing import TYPE_CHECKING, Any, Final, Optional, cast
import jinja2
import torch
from fastapi import Request
from PIL import Image
from pydantic import TypeAdapter
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
try:
import soundfile
except ImportError:
@@ -282,16 +286,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
lora_request=lora_request,
)
trace_headers = None if raw_request is None else await self._get_trace_headers(raw_request.headers)
generator = self.engine_client.generate(
prompt=engine_prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
output_modalities=output_modalities,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
)
generators.append(generator)
@@ -1678,10 +1677,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
):
choices: list[ChatCompletionResponseChoice] = []
final_res = omni_outputs.request_output
audio_data = final_res.multimodal_output.get("audio")
if stream:
audio_tensor = final_res.multimodal_output["audio"][-1].float().detach().cpu().numpy()
audio_tensor = audio_data[-1].float().detach().cpu().numpy()
else:
audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy()
if isinstance(audio_data, list):
audio_data = torch.cat(audio_data, dim=-1)
audio_tensor = audio_data.float().detach().cpu().numpy()
# Ensure audio is 1D (flatten if needed)
if audio_tensor.ndim > 1:
@@ -1924,29 +1926,30 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
logger.warning("Failed to decode reference image: %s", e)
# Build generation kwargs
gen_kwargs: dict[str, Any] = {
gen_prompt: OmniTextPrompt = {
"prompt": prompt,
"request_id": request_id,
"num_inference_steps": num_inference_steps,
"height": height,
"width": width,
"negative_prompt": negative_prompt,
"num_outputs_per_prompt": num_outputs_per_prompt,
"seed": seed,
}
gen_params = OmniDiffusionSamplingParams(
num_inference_steps=num_inference_steps,
height=height,
width=width,
num_outputs_per_prompt=num_outputs_per_prompt,
seed=seed,
)
if guidance_scale is not None:
gen_kwargs["guidance_scale"] = guidance_scale
gen_params.guidance_scale = guidance_scale
# Add Qwen-Image specific parameter
if true_cfg_scale is not None:
gen_kwargs["true_cfg_scale"] = true_cfg_scale
gen_params.true_cfg_scale = true_cfg_scale
# Add video generation parameters if set
if num_frames is not None:
gen_kwargs["num_frames"] = num_frames
gen_params.num_frames = num_frames
if guidance_scale_2 is not None:
gen_kwargs["guidance_scale_2"] = guidance_scale_2
gen_params.guidance_scale_2 = guidance_scale_2
# Parse per-request LoRA (works for both AsyncOmniDiffusion and AsyncOmni).
if lora_body and isinstance(lora_body, dict):
@@ -1969,16 +1972,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
lora_int_id = stable_lora_int_id(str(lora_path))
if lora_name and lora_path:
lora_req = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
gen_kwargs["lora_request"] = lora_req
gen_params.lora_request = lora_req
if lora_scale is not None:
gen_kwargs["lora_scale"] = float(lora_scale)
gen_params.lora_scale = float(lora_scale)
except Exception as e: # pragma: no cover - safeguard
logger.warning("Failed to parse LoRA request: %s", e)
# Add reference image if provided
if pil_images:
if len(pil_images) == 1:
gen_kwargs["pil_image"] = pil_images[0]
gen_prompt["multi_modal_data"] = {}
gen_prompt["multi_modal_data"]["image"] = pil_images[0]
else:
od_config = getattr(self._diffusion_engine, "od_config", None)
supports_multimodal_inputs = getattr(od_config, "supports_multimodal_inputs", False)
@@ -1986,7 +1990,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
# TODO: entry is asyncOmni. We hack the od config here.
supports_multimodal_inputs = True
if supports_multimodal_inputs:
gen_kwargs["pil_image"] = pil_images
gen_prompt["multi_modal_data"] = {}
gen_prompt["multi_modal_data"]["image"] = pil_images
else:
return self._create_error_response(
"Multiple input images are not supported by the current diffusion model. "
@@ -1999,18 +2004,24 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
# Handle both AsyncOmniDiffusion (returns OmniRequestOutput) and AsyncOmni (returns AsyncGenerator)
if hasattr(self._diffusion_engine, "stage_list"):
# AsyncOmni: iterate through async generator to get final output
diffusion_engine = cast(AsyncOmni, self._diffusion_engine)
result = None
async for output in self._diffusion_engine.generate(
prompt=gen_kwargs["prompt"],
request_id=gen_kwargs.get("request_id"),
sampling_params_list=[gen_kwargs], # Pass as single-stage params
async for output in diffusion_engine.generate(
prompt=gen_prompt,
sampling_params_list=[gen_params], # Pass as single-stage params
request_id=request_id,
):
result = output
if result is None:
return self._create_error_response("No output generated from AsyncOmni")
else:
# AsyncOmniDiffusion: direct call
result = await self._diffusion_engine.generate(**gen_kwargs)
diffusion_engine = cast(AsyncOmniDiffusion, self._diffusion_engine)
result = await diffusion_engine.generate(
prompt=gen_prompt,
sampling_params=gen_params,
request_id=request_id,
)
# Extract images from result
# Handle nested OmniRequestOutput structure where images might be in request_output
images = getattr(result.request_output, "images", [])
+3 -3
View File
@@ -57,12 +57,12 @@ def set_stage_devices(
- CUDA: Sets CUDA_VISIBLE_DEVICES and calls torch.cuda.set_device()
- NPU: Sets ASCEND_RT_VISIBLE_DEVICES and calls torch.npu.set_device()
"""
from vllm_omni.utils import detect_device_type, get_device_control_env_var
from vllm_omni.platforms import current_omni_platform
if device_type is None:
device_type = detect_device_type()
device_type = current_omni_platform.device_type
env_var = get_device_control_env_var()
env_var = current_omni_platform.device_control_env_var
try:
selected_physical: int | None = None
+6 -10
View File
@@ -10,7 +10,7 @@ from vllm.transformers_utils.config import get_config, get_hf_file_to_dict
from vllm.transformers_utils.repo_utils import file_or_path_exists
from vllm_omni.entrypoints.stage_utils import _to_dict
from vllm_omni.utils import detect_device_type, is_rocm
from vllm_omni.platforms import current_omni_platform
# Get the project root directory (2 levels up from this file)
PROJECT_ROOT = Path(__file__).parent.parent.parent
@@ -164,16 +164,12 @@ def resolve_model_config_path(model: str) -> str:
f"Model is not in standard transformers format and does not have model_index.json. "
f"Please ensure the model has proper configuration files with 'model_type' field"
)
device_type = detect_device_type()
# Try device-specific config first
if device_type != "cuda" or is_rocm():
device_config_file = f"vllm_omni/model_executor/stage_configs/{device_type}/{model_type}.yaml"
if is_rocm():
device_config_file = f"vllm_omni/model_executor/stage_configs/rocm/{model_type}.yaml"
device_config_path = PROJECT_ROOT / device_config_file
if os.path.exists(device_config_path):
return str(device_config_path)
default_config_path = current_omni_platform.get_default_stage_config_path()
model_type_str = f"{model_type}.yaml"
complete_config_path = PROJECT_ROOT / default_config_path / model_type_str
if os.path.exists(complete_config_path):
return str(complete_config_path)
# Fall back to default config
stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml"