Merge branch 'main' into dev/vllm-align
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
4b895459e4
196 files changed
+9406
-4132
No files matched your search
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import copy
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from collections.abc import AsyncGenerator, Iterable, Sequence
|
||||
from dataclasses import asdict
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
@@ -32,6 +33,7 @@ from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
get_final_stage_id_for_e2e,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams
|
||||
|
||||
# Internal imports (our code)
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
@@ -66,10 +68,8 @@ class AsyncOmni(OmniBase):
|
||||
asynchronous LLM and Diffusion models.
|
||||
|
||||
Args:
|
||||
*args: Variable length argument list.
|
||||
- args[0]: Model name or path to load.
|
||||
model: Model name or path to load.
|
||||
**kwargs: Arbitrary keyword arguments.
|
||||
- model: Model name or path to load (if not in args).
|
||||
- stage_configs_path: Optional path to YAML file containing stage
|
||||
configurations. If None, configurations are loaded from the model.
|
||||
- log_stats: Whether to enable statistics logging
|
||||
@@ -95,7 +95,7 @@ class AsyncOmni(OmniBase):
|
||||
... print(output)
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
|
||||
def __init__(self, model: str, **kwargs: dict[str, Any]) -> None:
|
||||
# Pause/resume control attributes
|
||||
self._pause_cond: asyncio.Condition = asyncio.Condition()
|
||||
self._paused: bool = False
|
||||
@@ -104,7 +104,7 @@ class AsyncOmni(OmniBase):
|
||||
self.request_states: dict[str, ClientRequestState] = {}
|
||||
self.output_handler: asyncio.Task | None = None
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
super().__init__(model, **kwargs)
|
||||
|
||||
# Register weak reference cleanup (called on garbage collection)
|
||||
self._weak_finalizer = weakref.finalize(
|
||||
@@ -163,6 +163,7 @@ class AsyncOmni(OmniBase):
|
||||
"vae_use_tiling": kwargs.get("vae_use_tiling", False),
|
||||
"cache_backend": cache_backend,
|
||||
"cache_config": cache_config,
|
||||
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
||||
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
|
||||
"enforce_eager": kwargs.get("enforce_eager", False),
|
||||
},
|
||||
@@ -233,7 +234,14 @@ class AsyncOmni(OmniBase):
|
||||
if hasattr(self, "_weak_finalizer"):
|
||||
self._weak_finalizer()
|
||||
|
||||
async def generate(self, *args: Any, **kwargs: dict[str, Any]) -> AsyncGenerator[OmniRequestOutput, None]:
|
||||
async def generate(
|
||||
self,
|
||||
prompt: OmniPromptType,
|
||||
request_id: str,
|
||||
sampling_params_list: Sequence[OmniSamplingParams] | None = None,
|
||||
*,
|
||||
output_modalities: list[str] | None = None,
|
||||
) -> AsyncGenerator[OmniRequestOutput, None]:
|
||||
"""Generate outputs for the given prompt asynchronously.
|
||||
|
||||
Coordinates multi-stage pipeline through YAML configuration.
|
||||
@@ -243,21 +251,13 @@ class AsyncOmni(OmniBase):
|
||||
sampling parameters from the sampling_params_list.
|
||||
|
||||
Args:
|
||||
*args: Arguments for generation.
|
||||
- prompt: Prompt to process. Can be a text string, token IDs,
|
||||
or multimodal prompt.
|
||||
- request_id: Unique identifier for this request
|
||||
- sampling_params_list: List of SamplingParams, one for each stage.
|
||||
Must have the same length as the number of stages.
|
||||
If None, uses default sampling params for each stage.
|
||||
**kwargs: Additional arguments for generation.
|
||||
- prompt: Prompt to process. Can be a text string, token IDs,
|
||||
or multimodal prompt.
|
||||
- request_id: Unique identifier for this request
|
||||
- sampling_params_list: List of SamplingParams, one for each stage.
|
||||
Must have the same length as the number of stages.
|
||||
If None, uses default sampling params for each stage.
|
||||
- output_modalities: Optional list of output modalities.
|
||||
prompt: Prompt to process. Can be a text string, token IDs,
|
||||
or multimodal prompt.
|
||||
request_id: Unique identifier for this request
|
||||
sampling_params_list: List of SamplingParams, one for each stage.
|
||||
Must have the same length as the number of stages.
|
||||
If None, uses default sampling params for each stage.
|
||||
output_modalities: Optional list of output modalities.
|
||||
|
||||
Yields:
|
||||
OmniRequestOutput objects as they are produced by each stage.
|
||||
@@ -276,33 +276,9 @@ class AsyncOmni(OmniBase):
|
||||
# Start output handler on the first call to generate()
|
||||
self._run_output_handler()
|
||||
|
||||
prompt = args[0] if args else kwargs.get("prompt")
|
||||
request_id = args[1] if len(args) > 1 else kwargs.get("request_id")
|
||||
sampling_params_list = args[2] if len(args) > 2 else kwargs.get("sampling_params_list")
|
||||
output_modalities = kwargs.get("output_modalities", None)
|
||||
# TODO: lora_request, trace_headers, priority are not supported yet
|
||||
|
||||
if sampling_params_list is None:
|
||||
# For Omni LLM, the params are parsed via the yaml file. For the current version,
|
||||
# diffusion params can parsed via the command line.
|
||||
omni_params_kwargs = {
|
||||
k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"]
|
||||
}
|
||||
|
||||
per_stage_params: list[Any] = []
|
||||
for stage_id, stage in enumerate(self.stage_list):
|
||||
stage_type = getattr(stage, "stage_type", "llm")
|
||||
if stage_type == "diffusion":
|
||||
default_dict = self.default_sampling_params_list[stage_id]
|
||||
# Merge user-provided kwargs
|
||||
merged = {**default_dict, **omni_params_kwargs}
|
||||
# Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest
|
||||
per_stage_params.append(merged)
|
||||
else:
|
||||
# LLM directly constructs SamplingParams, don't use the merged params
|
||||
per_stage_params.append(self.default_sampling_params_list[stage_id])
|
||||
|
||||
sampling_params_list = per_stage_params
|
||||
sampling_params_list = self.default_sampling_params_list
|
||||
|
||||
if len(sampling_params_list) != len(self.stage_list):
|
||||
raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}")
|
||||
@@ -329,38 +305,25 @@ class AsyncOmni(OmniBase):
|
||||
req_state = ClientRequestState(request_id)
|
||||
req_state.metrics = metrics
|
||||
self.request_states[request_id] = req_state
|
||||
|
||||
sp0: SamplingParams = sampling_params_list[0] # type: ignore[index]
|
||||
task = {
|
||||
"request_id": request_id,
|
||||
"engine_inputs": prompt,
|
||||
"sampling_params": sp0,
|
||||
}
|
||||
self.stage_list[0].submit(task)
|
||||
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
|
||||
_req_start_ts[request_id] = time.time()
|
||||
logger.info(
|
||||
f"[{self._name}] Entering scheduling loop: stages={num_stages}, final_stage={final_stage_id_for_e2e}"
|
||||
)
|
||||
if self.async_chunk:
|
||||
stage_queues = {stage_id: asyncio.Queue() for stage_id in range(num_stages)}
|
||||
req_state.stage_queues = stage_queues
|
||||
for i in range(num_stages):
|
||||
sp: SamplingParams = sampling_params_list[i]
|
||||
engine_inputs = prompt
|
||||
if i != 0:
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
prompt_1 = prompt.copy()
|
||||
prompt_1["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids)
|
||||
prompt_1["multi_modal_data"] = prompt_1["mm_processor_kwargs"] = None
|
||||
engine_inputs = prompt_1
|
||||
|
||||
task = {
|
||||
"request_id": request_id,
|
||||
"engine_inputs": engine_inputs,
|
||||
"sampling_params": sp,
|
||||
}
|
||||
self.stage_list[i].submit(task)
|
||||
metrics.stage_first_ts[i] = metrics.stage_first_ts[0] or time.time()
|
||||
|
||||
logger.info(f"[{self._name}] Enqueued request {request_id} to stage-{str(i)}")
|
||||
|
||||
_req_start_ts[request_id] = time.time()
|
||||
|
||||
logger.info(
|
||||
f"[{self._name}] Entering scheduling loop: "
|
||||
f"stages={num_stages}, final_stage={final_stage_id_for_e2e}"
|
||||
)
|
||||
async for output in self._process_async_results(
|
||||
request_id,
|
||||
prompt,
|
||||
sampling_params_list,
|
||||
req_state,
|
||||
metrics,
|
||||
final_stage_id_for_e2e,
|
||||
@@ -369,22 +332,6 @@ class AsyncOmni(OmniBase):
|
||||
):
|
||||
yield output
|
||||
else:
|
||||
sp0: SamplingParams = sampling_params_list[0] # type: ignore[index]
|
||||
task = {
|
||||
"request_id": request_id,
|
||||
"engine_inputs": prompt,
|
||||
"sampling_params": sp0,
|
||||
}
|
||||
self.stage_list[0].submit(task)
|
||||
|
||||
_req_start_ts[request_id] = time.time()
|
||||
# Mark first input time for stage-0
|
||||
metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
|
||||
logger.info(
|
||||
f"[{self._name}] Entering scheduling loop: "
|
||||
f"stages={num_stages}, final_stage={final_stage_id_for_e2e}"
|
||||
)
|
||||
|
||||
async for output in self._process_sequential_results(
|
||||
request_id,
|
||||
req_state,
|
||||
@@ -415,6 +362,8 @@ class AsyncOmni(OmniBase):
|
||||
async def _process_async_results(
|
||||
self,
|
||||
request_id: str,
|
||||
prompt: Any,
|
||||
sampling_params_list: list[SamplingParams],
|
||||
req_state: ClientRequestState,
|
||||
metrics: OrchestratorMetrics,
|
||||
final_stage_id_for_e2e: int,
|
||||
@@ -422,16 +371,34 @@ class AsyncOmni(OmniBase):
|
||||
wall_start_ts: float,
|
||||
) -> AsyncGenerator[OmniRequestOutput, None]:
|
||||
all_stages_finished = {stage_id: False for stage_id in range(final_stage_id_for_e2e + 1)}
|
||||
submit_flag = True
|
||||
while not all(all_stages_finished.values()):
|
||||
for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]):
|
||||
if all_stages_finished[stage_id]:
|
||||
continue
|
||||
result = await req_state.stage_queues[stage_id].get()
|
||||
logger.info(f"[{self._name}] Received result from stage-{stage_id}: {result}")
|
||||
try:
|
||||
result = req_state.stage_queues[stage_id].get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
await asyncio.sleep(0.001)
|
||||
continue
|
||||
|
||||
engine_outputs, finished, output_to_yield = self._process_single_result(
|
||||
result, stage, stage_id, metrics, req_start_ts, wall_start_ts, final_stage_id_for_e2e
|
||||
)
|
||||
|
||||
if submit_flag and stage_id == 0:
|
||||
submit_flag = False
|
||||
prompt_token_ids = engine_outputs.prompt_token_ids
|
||||
engine_input = copy.deepcopy(prompt)
|
||||
engine_input["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids)
|
||||
engine_input["multi_modal_data"] = engine_input["mm_processor_kwargs"] = None
|
||||
for i in range(1, len(self.stage_list)):
|
||||
task = {
|
||||
"request_id": request_id,
|
||||
"engine_inputs": engine_input,
|
||||
"sampling_params": sampling_params_list[i],
|
||||
}
|
||||
self.stage_list[i].submit(task)
|
||||
metrics.stage_first_ts[i] = time.time()
|
||||
all_stages_finished[stage_id] = finished
|
||||
|
||||
if output_to_yield:
|
||||
|
||||
@@ -12,16 +12,15 @@ import asyncio
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator, Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import fields
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -109,68 +108,19 @@ class AsyncOmniDiffusion:
|
||||
|
||||
logger.info("AsyncOmniDiffusion initialized with model: %s", model)
|
||||
|
||||
def _prepare_request(
|
||||
self,
|
||||
prompt: str,
|
||||
request_id: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniDiffusionRequest:
|
||||
"""Prepare a diffusion request from prompt and parameters.
|
||||
|
||||
Args:
|
||||
prompt: Text prompt for image generation
|
||||
request_id: Optional unique identifier for the request
|
||||
**kwargs: Additional generation parameters
|
||||
|
||||
Returns:
|
||||
OmniDiffusionRequest ready for processing
|
||||
"""
|
||||
if request_id is None:
|
||||
request_id = f"diff-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
field_names = {f.name for f in fields(OmniDiffusionRequest)}
|
||||
|
||||
init_kwargs = {
|
||||
"prompt": prompt,
|
||||
"request_id": request_id,
|
||||
}
|
||||
|
||||
for key, value in kwargs.items():
|
||||
if key in field_names:
|
||||
init_kwargs[key] = value
|
||||
|
||||
if "guidance_scale" in kwargs:
|
||||
init_kwargs["guidance_scale_provided"] = True
|
||||
|
||||
return OmniDiffusionRequest(**init_kwargs)
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
prompt: OmniPromptType,
|
||||
sampling_params: OmniDiffusionSamplingParams,
|
||||
request_id: str | None = None,
|
||||
num_inference_steps: int = 50,
|
||||
guidance_scale: float | None = None,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
negative_prompt: str | None = None,
|
||||
num_outputs_per_prompt: int = 1,
|
||||
seed: int | None = None,
|
||||
lora_request=None,
|
||||
**kwargs: Any,
|
||||
lora_request: LoRARequest | None = None,
|
||||
) -> OmniRequestOutput:
|
||||
"""Generate images asynchronously from a text prompt.
|
||||
|
||||
Args:
|
||||
prompt: Text prompt describing the desired image
|
||||
sampling_params: Sampling parameters
|
||||
request_id: Optional unique identifier for tracking the request
|
||||
num_inference_steps: Number of denoising steps (default: 50)
|
||||
guidance_scale: Classifier-free guidance scale (optional, uses model defaults if omitted)
|
||||
height: Optional image height in pixels
|
||||
width: Optional image width in pixels
|
||||
negative_prompt: Optional negative prompt for guidance
|
||||
num_outputs_per_prompt: Number of images to generate (default: 1)
|
||||
seed: Optional random seed for reproducibility
|
||||
**kwargs: Additional generation parameters
|
||||
|
||||
Returns:
|
||||
OmniRequestOutput containing generated images
|
||||
@@ -181,64 +131,38 @@ class AsyncOmniDiffusion:
|
||||
if request_id is None:
|
||||
request_id = f"diff-{uuid.uuid4().hex[:16]}"
|
||||
|
||||
# Prepare request
|
||||
request_kwargs = {
|
||||
"prompt": prompt,
|
||||
"request_id": request_id,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"negative_prompt": negative_prompt,
|
||||
"num_outputs_per_prompt": num_outputs_per_prompt,
|
||||
"seed": seed,
|
||||
"lora_request": lora_request,
|
||||
**kwargs,
|
||||
}
|
||||
if guidance_scale is not None:
|
||||
request_kwargs["guidance_scale"] = guidance_scale
|
||||
if sampling_params.guidance_scale:
|
||||
sampling_params.guidance_scale_provided = True
|
||||
|
||||
request = self._prepare_request(**request_kwargs)
|
||||
if lora_request is not None:
|
||||
sampling_params.lora_request = lora_request
|
||||
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=[prompt],
|
||||
sampling_params=sampling_params,
|
||||
request_ids=[request_id],
|
||||
)
|
||||
|
||||
logger.debug("Starting generation for request %s", request_id)
|
||||
|
||||
# Run engine in thread pool
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
# In async mode, only a single request is submitted at a time
|
||||
result = await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.engine.step,
|
||||
[request],
|
||||
request,
|
||||
)
|
||||
result = result[0]
|
||||
except Exception as e:
|
||||
logger.error("Generation failed for request %s: %s", request_id, e)
|
||||
raise RuntimeError(f"Diffusion generation failed: {e}") from e
|
||||
|
||||
# Check if result is already OmniRequestOutput
|
||||
if isinstance(result, OmniRequestOutput):
|
||||
# Update request_id if needed
|
||||
if not result.request_id:
|
||||
result.request_id = request_id
|
||||
return result
|
||||
|
||||
# Process results if not OmniRequestOutput
|
||||
images: list[Image.Image] = []
|
||||
if result is not None:
|
||||
if isinstance(result, list):
|
||||
for item in result:
|
||||
if isinstance(item, Image.Image):
|
||||
images.append(item)
|
||||
elif isinstance(result, Image.Image):
|
||||
images.append(result)
|
||||
|
||||
return OmniRequestOutput.from_diffusion(
|
||||
request_id=request_id,
|
||||
images=images,
|
||||
prompt=prompt,
|
||||
metrics={
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"guidance_scale": request.guidance_scale,
|
||||
},
|
||||
)
|
||||
# Update request_id if needed
|
||||
if not result.request_id:
|
||||
result.request_id = request_id
|
||||
return result
|
||||
|
||||
async def generate_stream(
|
||||
self,
|
||||
|
||||
@@ -175,6 +175,11 @@ class OmniServeCommand(CLISubcommand):
|
||||
default=None,
|
||||
help="JSON string of cache configuration (e.g., '{\"rel_l1_thresh\": 0.2}').",
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--enable-cache-dit-summary",
|
||||
action="store_true",
|
||||
help="Enable cache-dit summary logging after diffusion forward passes.",
|
||||
)
|
||||
|
||||
# VAE memory optimization parameters
|
||||
omni_config_group.add_argument(
|
||||
|
||||
@@ -10,14 +10,13 @@ from collections.abc import Callable, Generator, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import asdict
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
from typing import Any, Literal, overload
|
||||
|
||||
from omegaconf import OmegaConf
|
||||
from tqdm.auto import tqdm
|
||||
from vllm.inputs import PromptType
|
||||
from vllm import SamplingParams
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.distributed.omni_connectors import (
|
||||
get_stage_connector_config,
|
||||
initialize_orchestrator_connectors,
|
||||
@@ -42,6 +41,7 @@ from vllm_omni.entrypoints.utils import (
|
||||
load_stage_configs_from_yaml,
|
||||
resolve_model_config_path,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -82,10 +82,8 @@ class OmniBase:
|
||||
"""Base class for serving Omni models.
|
||||
|
||||
Args:
|
||||
*args: Variable length argument list.
|
||||
- args[0]: Model name or path to load.
|
||||
model: Model name or path to load.
|
||||
**kwargs: Arbitrary keyword arguments.
|
||||
- model: Model name or path to load (if not in args).
|
||||
- stage_configs_path: Optional path to YAML file containing stage
|
||||
configurations. If None, configurations are loaded from the model.
|
||||
- log_stats: Whether to enable statistics logging
|
||||
@@ -102,14 +100,9 @@ class OmniBase:
|
||||
- Additional keyword arguments passed to stage engines.
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
|
||||
model = args[0] if args else kwargs.get("model", "")
|
||||
assert model != "", "Null model id detected, please specify a model id."
|
||||
def __init__(self, model: str, **kwargs: Any) -> None:
|
||||
model = omni_snapshot_download(model)
|
||||
if args:
|
||||
args[0] = model
|
||||
elif kwargs.get("model", "") != "":
|
||||
kwargs["model"] = model
|
||||
kwargs["model"] = model
|
||||
|
||||
# Stage management attributes
|
||||
self.stage_list: list[OmniStage] = []
|
||||
@@ -502,10 +495,8 @@ class Omni(OmniBase):
|
||||
"""Unified entrypoint for both LLM and Diffusion models for better usability.
|
||||
|
||||
Args:
|
||||
*args: Variable length argument list.
|
||||
- args[0]: Model name or path to load.
|
||||
model: Model name or path to load.
|
||||
**kwargs: Arbitrary keyword arguments.
|
||||
- model: Model name or path to load (if not in args).
|
||||
- stage_configs_path: Optional path to YAML file containing stage
|
||||
configurations. If None, configurations are loaded from the model.
|
||||
- log_stats: Whether to enable statistics logging
|
||||
@@ -527,8 +518,8 @@ class Omni(OmniBase):
|
||||
>>> print(outputs)
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
def __init__(self, model: str, **kwargs: Any) -> None:
|
||||
super().__init__(model, **kwargs)
|
||||
|
||||
# Register weak reference cleanup (called on garbage collection)
|
||||
self._weak_finalizer = weakref.finalize(
|
||||
@@ -539,8 +530,31 @@ class Omni(OmniBase):
|
||||
self._ray_pg,
|
||||
)
|
||||
|
||||
@overload
|
||||
def generate(
|
||||
self, *args: Any, **kwargs: dict[str, Any]
|
||||
self,
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
|
||||
*,
|
||||
py_generator: Literal[True],
|
||||
) -> Generator[OmniRequestOutput, None, None]: ...
|
||||
|
||||
@overload
|
||||
def generate(
|
||||
self,
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
|
||||
*,
|
||||
py_generator: Literal[False] = False,
|
||||
) -> list[OmniRequestOutput]: ...
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None,
|
||||
*,
|
||||
py_generator: bool = False,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
) -> Generator[OmniRequestOutput, None, None] | list[OmniRequestOutput]:
|
||||
"""Generate outputs for the given prompts.
|
||||
|
||||
@@ -548,12 +562,10 @@ class Omni(OmniBase):
|
||||
Each stage will use OmniLLM or OmniDiffusion based on stage_type.
|
||||
|
||||
Args:
|
||||
*args: Variable length argument list.
|
||||
- args[0]: Input prompts for generation.
|
||||
- args[1]: Optional list of per-stage parameters.
|
||||
**kwargs: Arbitrary keyword arguments.
|
||||
- prompt: Input prompts for generation (if not in args).
|
||||
- sampling_params_list: Optional list of per-stage parameters (if not in args).
|
||||
prompts: Input prompt(s) for generation.
|
||||
sampling_params_list: Optional list of per-stage parameters.
|
||||
py_generator: Whether the returned result(s) are wrapped in a generator instead of a list.
|
||||
use_tqdm: Whether to use tqdm progress bar
|
||||
|
||||
Returns:
|
||||
List of OmniRequestOutput objects, one for each input prompt.
|
||||
@@ -563,40 +575,26 @@ class Omni(OmniBase):
|
||||
Raises:
|
||||
ValueError: If sampling_params_list is None or has incorrect length.
|
||||
"""
|
||||
prompts = args[0] if args else kwargs.get("prompts")
|
||||
sampling_params_list = args[1] if len(args) > 1 else kwargs.get("sampling_params_list")
|
||||
py_generator = kwargs.get("py_generator", False)
|
||||
if prompts is None:
|
||||
if kwargs.get("prompt") is None:
|
||||
raise ValueError("prompts is required for generation")
|
||||
prompts = kwargs.get("prompt")
|
||||
|
||||
if sampling_params_list is None:
|
||||
# For Omni LLM, the params are parsed via the yaml file. For the current version,
|
||||
# diffusion params can parsed via the command line.
|
||||
omni_params_kwargs = {
|
||||
k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"]
|
||||
}
|
||||
|
||||
per_stage_params: list[Any] = []
|
||||
for stage_id, stage in enumerate(self.stage_list):
|
||||
stage_type = getattr(stage, "stage_type", "llm")
|
||||
if stage_type == "diffusion":
|
||||
default_dict = self.default_sampling_params_list[stage_id]
|
||||
# Merge user-provided kwargs
|
||||
merged = {**default_dict, **omni_params_kwargs}
|
||||
# Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest
|
||||
per_stage_params.append(merged)
|
||||
sampling_params_list = self.default_sampling_params_list
|
||||
elif not isinstance(sampling_params_list, Sequence):
|
||||
# TODO: After the recent introduction of BAGEL model (one LLM and one Diffusion),
|
||||
# expect the text_to_image example code to run when only passing one OmniDiffusionSamplingParams
|
||||
# This behavior may be confusing, and future PR can improve it.
|
||||
per_stage_params: list[OmniSamplingParams] = []
|
||||
for default_stage_sp in self.default_sampling_params_list:
|
||||
default_sp_type = default_stage_sp.__class__
|
||||
if default_sp_type == sampling_params_list.__class__:
|
||||
per_stage_params.append(sampling_params_list)
|
||||
else:
|
||||
# LLM directly constructs SamplingParams, don't use the merged params
|
||||
per_stage_params.append(self.default_sampling_params_list[stage_id])
|
||||
|
||||
per_stage_params.append(default_stage_sp)
|
||||
sampling_params_list = per_stage_params
|
||||
|
||||
try:
|
||||
if py_generator:
|
||||
return self._run_generation_with_generator(prompts, sampling_params_list)
|
||||
else:
|
||||
outputs = list(self._run_generation(prompts, sampling_params_list))
|
||||
outputs = list(self._run_generation(prompts, sampling_params_list, use_tqdm))
|
||||
return outputs
|
||||
except Exception as e:
|
||||
logger.exception("[Orchestrator] Failed to run generation: %s", e)
|
||||
@@ -606,8 +604,8 @@ class Omni(OmniBase):
|
||||
|
||||
def _run_generation_with_generator(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest],
|
||||
sampling_params_list: Any | Sequence[Any] | None,
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params_list: Sequence[OmniSamplingParams],
|
||||
) -> Generator[OmniRequestOutput, None, None]:
|
||||
"""Run generation through all stages in the pipeline and return a generator."""
|
||||
gen = self._run_generation(prompts, sampling_params_list)
|
||||
@@ -622,8 +620,8 @@ class Omni(OmniBase):
|
||||
|
||||
def _run_generation(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest],
|
||||
sampling_params_list: Any | Sequence[Any] | None = None,
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params_list: Sequence[OmniSamplingParams],
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
) -> Generator[OmniRequestOutput, None, None]:
|
||||
"""Run generation through all stages in the pipeline."""
|
||||
@@ -631,18 +629,20 @@ class Omni(OmniBase):
|
||||
if sampling_params_list is None:
|
||||
raise ValueError("sampling_params_list is required for pipelined generation")
|
||||
|
||||
# Normalize sampling_params_list to a list
|
||||
if not isinstance(sampling_params_list, (list, tuple)):
|
||||
sampling_params_list = [sampling_params_list]
|
||||
else:
|
||||
sampling_params_list = list(sampling_params_list)
|
||||
|
||||
if len(sampling_params_list) != len(self.stage_list):
|
||||
raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}")
|
||||
|
||||
for i, (stage, sp) in enumerate(zip(self.stage_list, sampling_params_list)):
|
||||
ExpectedSPType = OmniDiffusionSamplingParams if stage.stage_type == "diffusion" else SamplingParams
|
||||
if not isinstance(sp, ExpectedSPType):
|
||||
raise ValueError(
|
||||
f"Expected sampling parameters with type {ExpectedSPType} in stage {i}, got {sp.__class__}"
|
||||
)
|
||||
|
||||
# Normalize prompts to a list for per-request iteration
|
||||
if not isinstance(prompts, (list, tuple)):
|
||||
request_prompts: list[PromptType] = [prompts]
|
||||
# str is also Sequence but only test list-like containers here
|
||||
if isinstance(prompts, str) or not isinstance(prompts, Sequence):
|
||||
request_prompts: list[OmniPromptType] = [prompts]
|
||||
else:
|
||||
request_prompts = list(prompts)
|
||||
|
||||
@@ -650,8 +650,8 @@ class Omni(OmniBase):
|
||||
num_stages = len(self.stage_list)
|
||||
|
||||
# Generate globally unique request IDs and map them to original prompts
|
||||
request_ids: list[str] = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))]
|
||||
request_id_to_prompt: dict[str, PromptType] = {rid: p for rid, p in zip(request_ids, request_prompts)}
|
||||
request_ids = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))]
|
||||
request_id_to_prompt = {rid: p for rid, p in zip(request_ids, request_prompts)}
|
||||
|
||||
# Track per-request start time for end-to-end timing
|
||||
_req_start_ts: dict[str, float] = {}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import fields
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
@@ -11,6 +11,8 @@ from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
# TODO configure logging properly
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
@@ -18,21 +20,6 @@ logging.basicConfig(level=logging.INFO)
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def prepare_requests(prompt: str | list[str], **kwargs):
|
||||
field_names = {f.name for f in fields(OmniDiffusionRequest)}
|
||||
|
||||
init_kwargs = {"prompt": prompt}
|
||||
|
||||
for key, value in kwargs.items():
|
||||
if key in field_names:
|
||||
init_kwargs[key] = value
|
||||
|
||||
if "guidance_scale" in kwargs:
|
||||
init_kwargs["guidance_scale_provided"] = True
|
||||
|
||||
return OmniDiffusionRequest(**init_kwargs)
|
||||
|
||||
|
||||
class OmniDiffusion:
|
||||
"""
|
||||
It is the main class to interact with vLLM-Omni diffusion models.
|
||||
@@ -100,42 +87,24 @@ class OmniDiffusion:
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
**kwargs,
|
||||
):
|
||||
prompts = []
|
||||
if isinstance(prompt, str):
|
||||
prompts.append(prompt)
|
||||
elif isinstance(prompt, list):
|
||||
prompts.extend(prompt)
|
||||
prompts: OmniPromptType | Sequence[OmniPromptType],
|
||||
sampling_params: OmniDiffusionSamplingParams,
|
||||
request_ids: list[str] = [],
|
||||
) -> list[OmniRequestOutput]:
|
||||
if isinstance(prompts, (str, dict)):
|
||||
prompts = [prompts]
|
||||
else:
|
||||
raise ValueError("Prompt must be a string or a list of strings")
|
||||
|
||||
requests: list[OmniDiffusionRequest] = []
|
||||
prompts = list(prompts)
|
||||
|
||||
# Check if request_id is provided in kwargs
|
||||
request_id = kwargs.get("request_id")
|
||||
request_ids = kwargs.pop("request_ids", None)
|
||||
if len(request_ids) < len(prompts):
|
||||
request_ids.extend(f"{i + len(request_ids)}_{uuid.uuid4()}" for i in range(len(prompts) - len(request_ids)))
|
||||
|
||||
for i, p in enumerate(prompts):
|
||||
req_kwargs = kwargs.copy()
|
||||
if request_ids and isinstance(request_ids, list) and i < len(request_ids):
|
||||
req_kwargs["request_id"] = request_ids[i]
|
||||
elif request_id is None:
|
||||
# Generate default ID consistent with OmniLLM: "{i}_{uuid}"
|
||||
req_kwargs["request_id"] = f"{i}_{uuid.uuid4()}"
|
||||
request = OmniDiffusionRequest(prompts, sampling_params, request_ids)
|
||||
return self._run_engine(request)
|
||||
|
||||
requests.append(
|
||||
prepare_requests(
|
||||
p,
|
||||
**req_kwargs,
|
||||
)
|
||||
)
|
||||
logger.info(f"Prepared {len(requests)} requests for generation.")
|
||||
return self._run_engine(requests)
|
||||
|
||||
def _run_engine(self, requests: list[OmniDiffusionRequest]):
|
||||
return self.engine.step(requests)
|
||||
def _run_engine(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]:
|
||||
return self.engine.step(request)
|
||||
|
||||
def close(self) -> None:
|
||||
self.engine.close()
|
||||
|
||||
+382
-354
@@ -15,9 +15,11 @@ import queue
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import fields
|
||||
from typing import Any
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
from vllm import PromptType, RequestOutput
|
||||
from vllm.inputs import TextPrompt
|
||||
from vllm.inputs.preprocess import InputPreprocessor
|
||||
from vllm.logger import init_logger
|
||||
@@ -31,6 +33,7 @@ from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.distributed.omni_connectors import build_stage_connectors
|
||||
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector
|
||||
from vllm_omni.distributed.omni_connectors.connectors.base import OmniConnectorBase
|
||||
from vllm_omni.distributed.ray_utils.utils import kill_ray_actor, start_ray_actor
|
||||
from vllm_omni.engine.arg_utils import AsyncOmniEngineArgs
|
||||
from vllm_omni.entrypoints.async_omni_diffusion import AsyncOmniDiffusion
|
||||
@@ -46,12 +49,29 @@ from vllm_omni.entrypoints.stage_utils import (
|
||||
maybe_dump_to_shm,
|
||||
set_stage_devices,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
from vllm_omni.utils import detect_device_type
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams, OmniTokensPrompt
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_worker_cls(engine_args: dict[str, Any]) -> None:
|
||||
worker_type = engine_args.pop("worker_type", None)
|
||||
if not worker_type:
|
||||
return
|
||||
if engine_args.get("worker_cls"):
|
||||
return
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
worker_type = str(worker_type).lower()
|
||||
if worker_type == "ar":
|
||||
engine_args["worker_cls"] = current_omni_platform.get_omni_ar_worker_cls()
|
||||
elif worker_type == "generation":
|
||||
engine_args["worker_cls"] = current_omni_platform.get_omni_generation_worker_cls()
|
||||
else:
|
||||
raise ValueError(f"Unknown worker_type: {worker_type}")
|
||||
|
||||
|
||||
def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]:
|
||||
"""Build OmniDiffusionConfig kwargs from engine args."""
|
||||
od_config = engine_args.get("od_config", {})
|
||||
@@ -64,35 +84,6 @@ def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]:
|
||||
return od_config
|
||||
|
||||
|
||||
def prepare_sampling_params(sampling_params: Any, stage_type: str) -> Any:
|
||||
"""Prepare sampling parameters for the given stage type.
|
||||
|
||||
Args:
|
||||
sampling_params: Raw sampling parameters (dict or SamplingParams)
|
||||
stage_type: Either "llm" or "diffusion"
|
||||
|
||||
Returns:
|
||||
Processed sampling parameters ready for engine consumption
|
||||
"""
|
||||
if stage_type == "diffusion":
|
||||
# For diffusion stages: extract kwargs, handling different input types
|
||||
if isinstance(sampling_params, dict):
|
||||
diffusion_kwargs = dict(sampling_params)
|
||||
else:
|
||||
diffusion_kwargs = getattr(sampling_params, "__dict__", {}) or {}
|
||||
|
||||
# Remove 'prompt' and 'request_id' to avoid conflict with explicit arguments
|
||||
diffusion_kwargs.pop("prompt", None)
|
||||
diffusion_kwargs.pop("request_id", None)
|
||||
return diffusion_kwargs
|
||||
|
||||
else: # stage_type == "llm"
|
||||
# For LLM stages: ensure we have a SamplingParams object
|
||||
if isinstance(sampling_params, dict):
|
||||
return SamplingParams(**sampling_params)
|
||||
return sampling_params
|
||||
|
||||
|
||||
class OmniStage:
|
||||
"""Stage manager for orchestrating a single stage in the omni pipeline.
|
||||
|
||||
@@ -123,7 +114,7 @@ class OmniStage:
|
||||
self.engine_outputs = None
|
||||
self.is_comprehension = getattr(stage_config, "is_comprehension", False)
|
||||
# Support for different stage types: "llm" (default) or "diffusion"
|
||||
self.stage_type = getattr(stage_config, "stage_type", "llm")
|
||||
self.stage_type: Literal["llm", "diffusion"] = getattr(stage_config, "stage_type", "llm")
|
||||
if hasattr(stage_config, "custom_process_input_func"):
|
||||
# Import the module specified in the config (already a full module path)
|
||||
module_path, func_name = stage_config.custom_process_input_func.rsplit(".", 1)
|
||||
@@ -137,7 +128,14 @@ class OmniStage:
|
||||
default_sampling_params = getattr(stage_config, "default_sampling_params", {})
|
||||
# For LLM stage, this can directly be a SamplingParams-compatible dict;
|
||||
# For diffusion stage, this only serves as default values for diffusion kwargs.
|
||||
self.default_sampling_params = _to_dict(default_sampling_params)
|
||||
default_sampling_params = _to_dict(default_sampling_params)
|
||||
# Further convert it to dataclass to check fields
|
||||
try:
|
||||
self.default_sampling_params = (
|
||||
SamplingParams if self.stage_type == "llm" else OmniDiffusionSamplingParams
|
||||
)(**default_sampling_params)
|
||||
except TypeError as error:
|
||||
raise TypeError(f"Invalid default_sampling_params for stage {self.stage_id}: {error}") from error
|
||||
# Runtime orchestration state (added)
|
||||
self._in_q: mp.Queue | None = None
|
||||
self._out_q: mp.Queue | None = None
|
||||
@@ -501,6 +499,9 @@ def _stage_worker(
|
||||
import os as _os
|
||||
import time as _time
|
||||
|
||||
from vllm_omni.plugins import load_omni_general_plugins
|
||||
|
||||
load_omni_general_plugins()
|
||||
# IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker /
|
||||
# GPUARModelRunner) are spawned with a fork-safe method.
|
||||
# Mooncake / gRPC / RDMA and CUDA/NCCL can deadlock under fork-with-threads.
|
||||
@@ -519,7 +520,10 @@ def _stage_worker(
|
||||
runtime_cfg = stage_payload.get("runtime", {})
|
||||
shm_threshold_bytes = int(stage_payload.get("shm_threshold_bytes", 65536))
|
||||
connectors_config = stage_payload.get("connectors_config", {})
|
||||
stage_type = stage_payload.get("stage_type", "llm")
|
||||
stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm")
|
||||
|
||||
if stage_type != "diffusion":
|
||||
_resolve_worker_cls(engine_args)
|
||||
|
||||
# Aggregates for running average
|
||||
_agg_total_tokens = 0
|
||||
@@ -530,7 +534,9 @@ def _stage_worker(
|
||||
# Device mapping
|
||||
device_type = None
|
||||
try:
|
||||
device_type = detect_device_type()
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
device_type = current_omni_platform.device_type
|
||||
set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type)
|
||||
except Exception as e:
|
||||
logger.warning("Device setup failed: %s", e)
|
||||
@@ -539,139 +545,137 @@ def _stage_worker(
|
||||
# when multiple instances start simultaneously
|
||||
# For TP/PP/DP/SP, we need to lock ALL devices that will be used by this stage
|
||||
lock_files = []
|
||||
if device_type == "cuda":
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
|
||||
if "parallel_config" in engine_args:
|
||||
parallel_config = engine_args["parallel_config"]
|
||||
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = parallel_config.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
|
||||
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
|
||||
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
|
||||
else:
|
||||
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = engine_args.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
|
||||
sequence_parallel_size = 1 # not use in omni model
|
||||
cfg_parallel_size = 1 # not used in omni model
|
||||
|
||||
if torch.cuda.is_available():
|
||||
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
|
||||
if "parallel_config" in engine_args:
|
||||
parallel_config = engine_args["parallel_config"]
|
||||
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = parallel_config.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = 1 # not used for diffusion
|
||||
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
|
||||
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
|
||||
else:
|
||||
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = engine_args.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
|
||||
sequence_parallel_size = 1 # not use in omni model
|
||||
cfg_parallel_size = 1 # not used in omni model
|
||||
# Calculate total number of devices needed for this stage
|
||||
# For a single stage worker:
|
||||
# - TP: splits model across devices (always needed)
|
||||
# - PP: splits layers across pipeline stages, but each stage uses TP devices
|
||||
# - DP: replicates model, but each replica uses TP devices
|
||||
# - PCP: context parallelism, typically uses TP devices
|
||||
# - SP: sequence parallelism, typically uses TP devices
|
||||
# - CFG: Classifier-Free Guidance parallelism for diffusion models
|
||||
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
|
||||
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
|
||||
num_devices_per_stage = (
|
||||
tensor_parallel_size
|
||||
* pipeline_parallel_size
|
||||
* data_parallel_size
|
||||
* prefill_context_parallel_size
|
||||
* sequence_parallel_size
|
||||
* cfg_parallel_size
|
||||
)
|
||||
|
||||
# Calculate total number of devices needed for this stage
|
||||
# For a single stage worker:
|
||||
# - TP: splits model across GPUs (always needed)
|
||||
# - PP: splits layers across pipelinestages, but each stage uses TP devices
|
||||
# - DP: replicates model, but each replica uses TP devices
|
||||
# - PCP: context parallelism, typically uses TP devices
|
||||
# - SP: sequence parallelism, typically uses TP devices
|
||||
# - CFG: Classifier-Free Guidance parallelism for diffusion models
|
||||
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
|
||||
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
|
||||
num_devices_per_stage = (
|
||||
tensor_parallel_size
|
||||
* pipeline_parallel_size
|
||||
* data_parallel_size
|
||||
* prefill_context_parallel_size
|
||||
* sequence_parallel_size
|
||||
* cfg_parallel_size
|
||||
)
|
||||
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
|
||||
# After set_stage_devices, this env var is set to physical device(s)
|
||||
device_control_env = current_omni_platform.device_control_env_var
|
||||
visible_devices_str = _os.environ.get(device_control_env)
|
||||
physical_devices = []
|
||||
|
||||
# Get physical device IDs from CUDA_VISIBLE_DEVICES
|
||||
# After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s)
|
||||
cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
physical_devices = []
|
||||
if visible_devices_str:
|
||||
try:
|
||||
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
if cuda_visible_devices:
|
||||
if not physical_devices:
|
||||
# Fallback: use logical device count if device control env var not set
|
||||
num_devices = current_omni_platform.get_device_count()
|
||||
physical_devices = list(range(num_devices))
|
||||
|
||||
# Determine which devices will be used (min of devices per stage and available devices)
|
||||
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
|
||||
devices_to_lock = physical_devices[:num_devices_to_lock]
|
||||
|
||||
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
|
||||
devices_to_lock = sorted(devices_to_lock)
|
||||
|
||||
logger.debug(
|
||||
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
|
||||
tensor_parallel_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
prefill_context_parallel_size,
|
||||
sequence_parallel_size,
|
||||
cfg_parallel_size,
|
||||
num_devices_to_lock,
|
||||
devices_to_lock,
|
||||
)
|
||||
|
||||
# Acquire exclusive locks for all devices using fcntl.flock
|
||||
# Locks are automatically released when process dies
|
||||
wait_start = _time.time()
|
||||
acquired_lock_fds = [] # Store file descriptors to keep locks alive
|
||||
|
||||
for device_id in devices_to_lock:
|
||||
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
|
||||
lock_acquired = False
|
||||
|
||||
while not lock_acquired:
|
||||
try:
|
||||
# Open or create the lock file
|
||||
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
|
||||
|
||||
# Try to acquire exclusive lock (non-blocking first)
|
||||
try:
|
||||
physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()]
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
# Successfully acquired lock - write PID
|
||||
_os.ftruncate(lock_fd, 0) # Clear file
|
||||
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
|
||||
_os.fsync(lock_fd) # Ensure written to disk
|
||||
lock_acquired = True
|
||||
acquired_lock_fds.append(lock_fd)
|
||||
logger.debug("Acquired exclusive lock for device %s", device_id)
|
||||
except BlockingIOError:
|
||||
# Lock is held by another process
|
||||
_os.close(lock_fd)
|
||||
|
||||
if not physical_devices:
|
||||
# Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set
|
||||
num_devices = torch.cuda.device_count()
|
||||
physical_devices = list(range(num_devices))
|
||||
|
||||
# Determine which devices will be used (min of devices per stage and available devices)
|
||||
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
|
||||
devices_to_lock = physical_devices[:num_devices_to_lock]
|
||||
|
||||
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
|
||||
devices_to_lock = sorted(devices_to_lock)
|
||||
|
||||
logger.debug(
|
||||
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d; will lock %d devices: %s",
|
||||
tensor_parallel_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
prefill_context_parallel_size,
|
||||
sequence_parallel_size,
|
||||
num_devices_to_lock,
|
||||
devices_to_lock,
|
||||
)
|
||||
|
||||
# Acquire exclusive locks for all devices using fcntl.flock
|
||||
# Locks are automatically released when process dies
|
||||
wait_start = _time.time()
|
||||
acquired_lock_fds = [] # Store file descriptors to keep locks alive
|
||||
|
||||
for device_id in devices_to_lock:
|
||||
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
|
||||
lock_acquired = False
|
||||
|
||||
while not lock_acquired:
|
||||
try:
|
||||
# Open or create the lock file
|
||||
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
|
||||
|
||||
# Try to acquire exclusive lock (non-blocking first)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
# Successfully acquired lock - write PID
|
||||
_os.ftruncate(lock_fd, 0) # Clear file
|
||||
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
|
||||
_os.fsync(lock_fd) # Ensure written to disk
|
||||
lock_acquired = True
|
||||
acquired_lock_fds.append(lock_fd)
|
||||
logger.debug("Acquired exclusive lock for device %s", device_id)
|
||||
except BlockingIOError:
|
||||
# Lock is held by another process
|
||||
_os.close(lock_fd)
|
||||
|
||||
# Check if we've been waiting too long
|
||||
if _time.time() - wait_start > stage_init_timeout:
|
||||
logger.warning(
|
||||
"Timeout waiting for device %s initialization lock, proceeding anyway",
|
||||
device_id,
|
||||
)
|
||||
break
|
||||
|
||||
# Wait a bit before retrying
|
||||
_time.sleep(0.1)
|
||||
except OSError as e:
|
||||
# Other error - log and continue without lock
|
||||
logger.debug(
|
||||
"Failed to acquire lock for device %s: %s, continuing anyway",
|
||||
# Check if we've been waiting too long
|
||||
if _time.time() - wait_start > stage_init_timeout:
|
||||
logger.warning(
|
||||
"Timeout waiting for device %s initialization lock, proceeding anyway",
|
||||
device_id,
|
||||
e,
|
||||
)
|
||||
try:
|
||||
_os.close(lock_fd)
|
||||
except (OSError, NameError):
|
||||
pass
|
||||
break
|
||||
|
||||
lock_files = acquired_lock_fds
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"[Stage-%s] Failed to set up sequential initialization lock: %s",
|
||||
stage_id,
|
||||
e,
|
||||
)
|
||||
# Wait a bit before retrying
|
||||
_time.sleep(0.1)
|
||||
except OSError as e:
|
||||
# Other error - log and continue without lock
|
||||
logger.debug(
|
||||
"Failed to acquire lock for device %s: %s, continuing anyway",
|
||||
device_id,
|
||||
e,
|
||||
)
|
||||
try:
|
||||
_os.close(lock_fd)
|
||||
except (OSError, NameError):
|
||||
pass
|
||||
break
|
||||
|
||||
lock_files = acquired_lock_fds
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"[Stage-%s] Failed to set up sequential initialization lock: %s",
|
||||
stage_id,
|
||||
e,
|
||||
)
|
||||
# Init engine based on stage_type
|
||||
logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys()))
|
||||
if engine_args.get("async_chunk", False):
|
||||
@@ -708,15 +712,14 @@ def _stage_worker(
|
||||
pass
|
||||
logger.debug("Engine initialized")
|
||||
# Initialize OmniConnectors if configured
|
||||
connectors = {}
|
||||
connectors: dict[tuple[str, str], OmniConnectorBase] | None = {}
|
||||
if connectors_config:
|
||||
built_connectors = build_stage_connectors(
|
||||
connectors = build_stage_connectors(
|
||||
stage_id=stage_id,
|
||||
connectors_config=connectors_config,
|
||||
)
|
||||
if built_connectors is None:
|
||||
if connectors is None:
|
||||
return
|
||||
connectors = built_connectors
|
||||
|
||||
# Signal readiness to orchestrator
|
||||
try:
|
||||
@@ -785,6 +788,7 @@ def _stage_worker(
|
||||
continue
|
||||
|
||||
batch_tasks: list[dict[str, Any]] = [task]
|
||||
tasks_failed_to_add_to_batch: list[dict[str, Any]] = []
|
||||
start_time = _time.time()
|
||||
if max_batch_size > 1:
|
||||
while len(batch_tasks) < max_batch_size:
|
||||
@@ -800,7 +804,20 @@ def _stage_worker(
|
||||
if extra_type == OmniStageTaskType.PROFILER_STOP:
|
||||
out_q.put({"type": "profiler_result", "data": p_data})
|
||||
continue
|
||||
batch_tasks.append(extra)
|
||||
# Ensure that all tasks have the same sampling params
|
||||
# If no, put them in a temporary container and add back to queue
|
||||
# This should be always true, because user only calls omni.generate() once and it blocks
|
||||
# User can only pass one sampling param object, but the list of prompts are separated.
|
||||
if task.get("sampling_params") != extra.get("sampling_params"):
|
||||
logger.warning(
|
||||
"""In offline mode, expect all prompts in one `omni.generate()` call to share same sampling params""" # noqa: E501 # line too long
|
||||
f"""However, prompt {task.get("engine_inputs")} has sampling params {task.get("sampling_params")}, """ # noqa: E501 # line too long
|
||||
f"""whereas the prompt {extra.get("engine_inputs")} has sampling params {extra.get("sampling_params")}.""" # noqa: E501 # line too long
|
||||
"""The two tasks cannot be combined in one batch request."""
|
||||
)
|
||||
tasks_failed_to_add_to_batch.append(extra)
|
||||
else:
|
||||
batch_tasks.append(extra)
|
||||
end_time = _time.time()
|
||||
duration = end_time - start_time
|
||||
if duration > batch_timeout:
|
||||
@@ -815,9 +832,13 @@ def _stage_worker(
|
||||
break
|
||||
else:
|
||||
continue
|
||||
for task_to_readd in tasks_failed_to_add_to_batch:
|
||||
in_q.put(task_to_readd)
|
||||
# Ensure that the popped tasks are with identical sampling params. Take one of them.
|
||||
batch_engine_sampling_params: OmniSamplingParams = batch_tasks[0]["sampling_params"]
|
||||
|
||||
batch_request_ids: list[Any] = []
|
||||
batch_engine_inputs: list[Any] = []
|
||||
batch_engine_inputs: list[OmniPromptType] = []
|
||||
_rx_bytes_by_rid: dict[Any, int] = {}
|
||||
_rx_decode_ms_by_rid: dict[Any, float] = {}
|
||||
_in_flight_ms_by_rid: dict[Any, float] = {}
|
||||
@@ -839,6 +860,9 @@ def _stage_worker(
|
||||
connectors=connectors,
|
||||
stage_id=stage_id,
|
||||
)
|
||||
# TODO: hack type annotation for now.
|
||||
# A better way is to refine type annotation of connection and task/payloads, maybe using template types.
|
||||
ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein)
|
||||
|
||||
if ein is None or _rx_metrics is None:
|
||||
raise RuntimeError(
|
||||
@@ -850,17 +874,14 @@ def _stage_worker(
|
||||
_rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
|
||||
|
||||
batch_request_ids.append(rid)
|
||||
if isinstance(ein, list):
|
||||
if isinstance(ein, (str, dict)):
|
||||
# Types like OmniTextPrompt, TextPrompt are TypedDict, essentially dict and enters this branch
|
||||
batch_engine_inputs.append(ein)
|
||||
elif isinstance(ein, Sequence):
|
||||
batch_engine_inputs.extend(ein)
|
||||
elif isinstance(ein, dict):
|
||||
batch_engine_inputs.append(ein)
|
||||
elif isinstance(ein, str):
|
||||
# For diffusion stage-0, ein might be a string prompt directly
|
||||
batch_engine_inputs.append(ein)
|
||||
else:
|
||||
# For other types (e.g., OmniTokensPrompt, TextPrompt), append as-is
|
||||
# Other unknown types, append as-is
|
||||
batch_engine_inputs.append(ein)
|
||||
sampling_params = batch_tasks[0]["sampling_params"]
|
||||
logger.debug(
|
||||
"Received batch size=%d, request_ids=%s",
|
||||
len(batch_tasks),
|
||||
@@ -868,65 +889,30 @@ def _stage_worker(
|
||||
)
|
||||
try:
|
||||
_batch_seq += 1
|
||||
gen_outputs: list[Any] = []
|
||||
gen_outputs: list[OmniRequestOutput | RequestOutput] = []
|
||||
_gen_t0 = _time.time()
|
||||
if stage_type == "diffusion":
|
||||
# For diffusion, batch_engine_inputs should be prompts (strings)
|
||||
# Convert to list of strings if needed
|
||||
prompts = []
|
||||
for ein in batch_engine_inputs:
|
||||
if isinstance(ein, str):
|
||||
prompts.append(ein)
|
||||
elif isinstance(ein, dict) and "prompt" in ein:
|
||||
prompts.append(ein["prompt"])
|
||||
elif hasattr(ein, "prompt"):
|
||||
prompts.append(ein.prompt)
|
||||
else:
|
||||
prompts.append(str(ein))
|
||||
# Prepare diffusion kwargs from sampling parameters
|
||||
diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion")
|
||||
|
||||
# Pass batch_request_ids to ensure correct ID mapping
|
||||
diffusion_kwargs["request_ids"] = batch_request_ids
|
||||
|
||||
stage_engine = cast(OmniDiffusion, stage_engine)
|
||||
batch_engine_sampling_params = cast(OmniDiffusionSamplingParams, batch_engine_sampling_params)
|
||||
# Diffusion generate returns results directly, not an iterator
|
||||
diffusion_results = stage_engine.generate(prompts, **diffusion_kwargs)
|
||||
# Convert to list format compatible with LLM outputs
|
||||
# Ensure each result has a request_id for proper mapping
|
||||
if isinstance(diffusion_results, list):
|
||||
gen_outputs = diffusion_results
|
||||
# Assign request_ids if not present
|
||||
for idx, result in enumerate(gen_outputs):
|
||||
if not hasattr(result, "request_id") or result.request_id is None:
|
||||
if idx < len(batch_request_ids):
|
||||
if hasattr(result, "request_id"):
|
||||
result.request_id = batch_request_ids[idx]
|
||||
else:
|
||||
# Create a wrapper object if result doesn't support request_id
|
||||
from types import SimpleNamespace
|
||||
|
||||
wrapped = SimpleNamespace()
|
||||
wrapped.request_id = batch_request_ids[idx]
|
||||
wrapped.output = result
|
||||
gen_outputs[idx] = wrapped
|
||||
else:
|
||||
gen_outputs = [diffusion_results]
|
||||
# Assign request_id to single result
|
||||
if len(batch_request_ids) > 0:
|
||||
if hasattr(gen_outputs[0], "request_id"):
|
||||
gen_outputs[0].request_id = batch_request_ids[0]
|
||||
else:
|
||||
from types import SimpleNamespace
|
||||
|
||||
wrapped = SimpleNamespace()
|
||||
wrapped.request_id = batch_request_ids[0]
|
||||
wrapped.output = gen_outputs[0]
|
||||
gen_outputs[0] = wrapped
|
||||
diffusion_results = stage_engine.generate(
|
||||
batch_engine_inputs, batch_engine_sampling_params, batch_request_ids
|
||||
)
|
||||
gen_outputs.extend(diffusion_results)
|
||||
# Assign request_ids if not present
|
||||
for idx, result in enumerate(gen_outputs):
|
||||
if not hasattr(result, "request_id") or result.request_id is None:
|
||||
if idx < len(batch_request_ids):
|
||||
result.request_id = batch_request_ids[idx]
|
||||
else:
|
||||
# LLM engine: use vLLM native SamplingParams
|
||||
llm_sampling_params = prepare_sampling_params(sampling_params, "llm")
|
||||
for ro in stage_engine.generate(batch_engine_inputs, llm_sampling_params, use_tqdm=False):
|
||||
gen_outputs.append(ro)
|
||||
stage_engine = cast(OmniLLM, stage_engine)
|
||||
batch_engine_sampling_params = cast(SamplingParams, batch_engine_sampling_params)
|
||||
results = stage_engine.generate(
|
||||
batch_engine_inputs, # type: ignore # silent complaints about list of subclassed TypedDict
|
||||
batch_engine_sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
gen_outputs.extend(results)
|
||||
_gen_t1 = _time.time()
|
||||
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
|
||||
logger.debug(f"Generate done: batch={len(batch_tasks)}, req_ids={batch_request_ids}, gen_ms={_gen_ms:.1f}")
|
||||
@@ -935,7 +921,7 @@ def _stage_worker(
|
||||
req_to_outputs: dict[Any, list[Any]] = {rid: [] for rid in batch_request_ids}
|
||||
unmapped: list[Any] = []
|
||||
for ro in gen_outputs:
|
||||
rid = getattr(ro, "request_id", None)
|
||||
rid = ro.request_id
|
||||
if rid in req_to_outputs:
|
||||
req_to_outputs[rid].append(ro)
|
||||
else:
|
||||
@@ -1037,6 +1023,9 @@ async def _stage_worker_async(
|
||||
import os as _os
|
||||
import time as _time
|
||||
|
||||
from vllm_omni.plugins import load_omni_general_plugins
|
||||
|
||||
load_omni_general_plugins()
|
||||
# IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker /
|
||||
# GPUARModelRunner) are spawned with a fork-safe method.
|
||||
if _os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn":
|
||||
@@ -1054,6 +1043,9 @@ async def _stage_worker_async(
|
||||
connectors_config = stage_payload.get("connectors_config", {})
|
||||
stage_type = stage_payload.get("stage_type", "llm")
|
||||
|
||||
if stage_type != "diffusion":
|
||||
_resolve_worker_cls(engine_args)
|
||||
|
||||
in_q = omni_stage._in_q
|
||||
out_q = omni_stage._out_q
|
||||
|
||||
@@ -1067,9 +1059,9 @@ async def _stage_worker_async(
|
||||
# Device mapping
|
||||
device_type = None
|
||||
try:
|
||||
from vllm_omni.utils import detect_device_type
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
device_type = detect_device_type()
|
||||
device_type = current_omni_platform.device_type
|
||||
set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type)
|
||||
except Exception as e:
|
||||
logger.warning("Device setup failed: %s", e)
|
||||
@@ -1087,120 +1079,136 @@ async def _stage_worker_async(
|
||||
|
||||
# Sequential initialization on the same device to avoid memory calculation errors
|
||||
# when multiple instances start simultaneously
|
||||
# For TP, we need to lock ALL devices that will be used by this stage
|
||||
# For TP/PP/DP/PCP, we need to lock ALL devices that will be used by this stage
|
||||
lock_files = []
|
||||
if device_type == "cuda":
|
||||
try:
|
||||
import torch
|
||||
try:
|
||||
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
|
||||
if "parallel_config" in engine_args:
|
||||
parallel_config = engine_args["parallel_config"]
|
||||
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = parallel_config.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
|
||||
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
|
||||
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
|
||||
else:
|
||||
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = engine_args.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
|
||||
sequence_parallel_size = 1 # not use in omni model
|
||||
cfg_parallel_size = 1 # not used in omni model
|
||||
|
||||
if torch.cuda.is_available():
|
||||
# Get all parallel sizes from engine_args (defaults to 1)
|
||||
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
|
||||
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
|
||||
data_parallel_size = engine_args.get("data_parallel_size", 1)
|
||||
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
|
||||
# Calculate total number of devices needed for this stage
|
||||
# For a single stage worker:
|
||||
# - TP: splits model across devices (always needed)
|
||||
# - PP: splits layers across pipeline stages, but each stage uses TP devices
|
||||
# - DP: replicates model, but each replica uses TP devices
|
||||
# - PCP: context parallelism, typically uses TP devices
|
||||
# - SP: sequence parallelism, typically uses TP devices
|
||||
# - CFG: Classifier-Free Guidance parallelism for diffusion models
|
||||
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
|
||||
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
|
||||
num_devices_per_stage = (
|
||||
tensor_parallel_size
|
||||
* pipeline_parallel_size
|
||||
* data_parallel_size
|
||||
* prefill_context_parallel_size
|
||||
* sequence_parallel_size
|
||||
* cfg_parallel_size
|
||||
)
|
||||
|
||||
# Calculate total number of devices needed for this stage
|
||||
# For a single stage worker in omni:
|
||||
# - TP: splits model across GPUs (always needed)
|
||||
# - PP: splits layers across stages, but each stage uses TP devices
|
||||
# - DP: replicates model, but each replica uses TP devices
|
||||
# - PCP: context parallelism, typically uses TP devices
|
||||
# The number of devices per stage is determined by TP * PP * DP * PCP size
|
||||
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
|
||||
num_devices_per_stage = (
|
||||
tensor_parallel_size * pipeline_parallel_size * data_parallel_size * prefill_context_parallel_size
|
||||
)
|
||||
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
|
||||
# After set_stage_devices, this env var is set to physical device(s)
|
||||
device_control_env = current_omni_platform.device_control_env_var
|
||||
visible_devices_str = _os.environ.get(device_control_env)
|
||||
physical_devices = []
|
||||
|
||||
# Get physical device IDs from CUDA_VISIBLE_DEVICES
|
||||
# After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s)
|
||||
cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
physical_devices = []
|
||||
if visible_devices_str:
|
||||
try:
|
||||
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
|
||||
if cuda_visible_devices:
|
||||
if not physical_devices:
|
||||
# Fallback: use logical device count if device control env var not set
|
||||
num_devices = current_omni_platform.get_device_count()
|
||||
physical_devices = list(range(num_devices))
|
||||
|
||||
# Determine which devices will be used (min of devices per stage and available devices)
|
||||
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
|
||||
devices_to_lock = physical_devices[:num_devices_to_lock]
|
||||
|
||||
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
|
||||
devices_to_lock = sorted(devices_to_lock)
|
||||
|
||||
logger.debug(
|
||||
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
|
||||
tensor_parallel_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
prefill_context_parallel_size,
|
||||
sequence_parallel_size,
|
||||
cfg_parallel_size,
|
||||
num_devices_to_lock,
|
||||
devices_to_lock,
|
||||
)
|
||||
|
||||
# Acquire exclusive locks for all devices using fcntl.flock
|
||||
# Locks are automatically released when process dies
|
||||
wait_start = _time.time()
|
||||
acquired_lock_fds = [] # Store file descriptors to keep locks alive
|
||||
|
||||
for device_id in devices_to_lock:
|
||||
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
|
||||
lock_acquired = False
|
||||
|
||||
while not lock_acquired:
|
||||
try:
|
||||
# Open or create the lock file
|
||||
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
|
||||
|
||||
# Try to acquire exclusive lock (non-blocking first)
|
||||
try:
|
||||
physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()]
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
# Successfully acquired lock - write PID
|
||||
_os.ftruncate(lock_fd, 0) # Clear file
|
||||
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
|
||||
_os.fsync(lock_fd) # Ensure written to disk
|
||||
lock_acquired = True
|
||||
acquired_lock_fds.append(lock_fd)
|
||||
logger.debug("Acquired exclusive lock for device %s", device_id)
|
||||
except BlockingIOError:
|
||||
# Lock is held by another process
|
||||
_os.close(lock_fd)
|
||||
|
||||
if not physical_devices:
|
||||
# Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set
|
||||
num_devices = torch.cuda.device_count()
|
||||
physical_devices = list(range(num_devices))
|
||||
|
||||
# Determine which devices will be used (min of devices per stage and available devices)
|
||||
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
|
||||
devices_to_lock = physical_devices[:num_devices_to_lock]
|
||||
|
||||
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
|
||||
devices_to_lock = sorted(devices_to_lock)
|
||||
|
||||
logger.debug(
|
||||
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d; will lock %d devices: %s",
|
||||
tensor_parallel_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
prefill_context_parallel_size,
|
||||
num_devices_to_lock,
|
||||
devices_to_lock,
|
||||
)
|
||||
|
||||
# Acquire exclusive locks for all devices using fcntl.flock
|
||||
# Locks are automatically released when process dies
|
||||
wait_start = _time.time()
|
||||
acquired_lock_fds = [] # Store file descriptors to keep locks alive
|
||||
|
||||
for device_id in devices_to_lock:
|
||||
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
|
||||
lock_acquired = False
|
||||
|
||||
while not lock_acquired:
|
||||
try:
|
||||
# Open or create the lock file
|
||||
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
|
||||
|
||||
# Try to acquire exclusive lock (non-blocking first)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
# Successfully acquired lock - write PID
|
||||
_os.ftruncate(lock_fd, 0) # Clear file
|
||||
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
|
||||
_os.fsync(lock_fd) # Ensure written to disk
|
||||
lock_acquired = True
|
||||
acquired_lock_fds.append(lock_fd)
|
||||
logger.debug("Acquired exclusive lock for device %s", device_id)
|
||||
except BlockingIOError:
|
||||
# Lock is held by another process
|
||||
_os.close(lock_fd)
|
||||
|
||||
# Check if we've been waiting too long
|
||||
if _time.time() - wait_start > stage_init_timeout:
|
||||
logger.warning(
|
||||
"Timeout waiting for device %s initialization lock, "
|
||||
"proceeding anyway with timeout %s",
|
||||
device_id,
|
||||
stage_init_timeout,
|
||||
)
|
||||
break
|
||||
|
||||
# Wait a bit before retrying
|
||||
_time.sleep(0.1)
|
||||
except OSError as e:
|
||||
# Other error - log and continue without lock
|
||||
logger.debug(
|
||||
"Failed to acquire lock for device %s: %s, continuing anyway",
|
||||
# Check if we've been waiting too long
|
||||
if _time.time() - wait_start > stage_init_timeout:
|
||||
logger.warning(
|
||||
"Timeout waiting for device %s initialization lock, proceeding anyway with timeout %s",
|
||||
device_id,
|
||||
e,
|
||||
stage_init_timeout,
|
||||
)
|
||||
try:
|
||||
_os.close(lock_fd)
|
||||
except (OSError, NameError):
|
||||
pass
|
||||
break
|
||||
|
||||
lock_files = acquired_lock_fds
|
||||
except Exception as e:
|
||||
logger.debug("Failed to set up sequential initialization lock: %s", e)
|
||||
# Wait a bit before retrying
|
||||
_time.sleep(0.1)
|
||||
except OSError as e:
|
||||
# Other error - log and continue without lock
|
||||
logger.debug(
|
||||
"Failed to acquire lock for device %s: %s, continuing anyway",
|
||||
device_id,
|
||||
e,
|
||||
)
|
||||
try:
|
||||
_os.close(lock_fd)
|
||||
except (OSError, NameError):
|
||||
pass
|
||||
break
|
||||
|
||||
lock_files = acquired_lock_fds
|
||||
except Exception as e:
|
||||
logger.debug("Failed to set up sequential initialization lock: %s", e)
|
||||
|
||||
# Init engine based on stage_type
|
||||
logger.debug(
|
||||
@@ -1354,6 +1362,10 @@ async def _stage_worker_async(
|
||||
connectors=connectors,
|
||||
stage_id=stage_id,
|
||||
)
|
||||
# TODO: hack type annotation for now.
|
||||
# A better way is to refine type annotation of connection and task/payloads, maybe using template types.
|
||||
ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein)
|
||||
|
||||
if ein is None or _rx_metrics is None:
|
||||
raise RuntimeError(
|
||||
f"[Stage-{stage_id}] Missing connector payload for request {rid}. "
|
||||
@@ -1362,36 +1374,23 @@ async def _stage_worker_async(
|
||||
_rx_decode_ms_by_rid[rid] = float(_rx_metrics.get("rx_decode_time_ms", 0.0))
|
||||
_rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
|
||||
|
||||
sampling_params = task["sampling_params"]
|
||||
logger.debug("Received batch size=1, request_ids=%s", rid)
|
||||
_gen_t0 = _time.time()
|
||||
if isinstance(ein, list):
|
||||
if isinstance(ein, Sequence) and not isinstance(ein, str):
|
||||
ein = ein[0]
|
||||
|
||||
if stage_type == "diffusion":
|
||||
# For diffusion, ein should be prompts (strings)
|
||||
# Convert to string if needed
|
||||
if isinstance(ein, str):
|
||||
prompt = ein
|
||||
elif isinstance(ein, dict) and "prompt" in ein:
|
||||
prompt = ein["prompt"]
|
||||
elif hasattr(ein, "prompt"):
|
||||
prompt = ein.prompt
|
||||
else:
|
||||
prompt = str(ein)
|
||||
|
||||
# Prepare diffusion kwargs from sampling parameters
|
||||
diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion")
|
||||
diffusion_sampling_params = cast(OmniDiffusionSamplingParams, task["sampling_params"])
|
||||
# AsyncOmniDiffusion.generate returns a single result, not an async generator
|
||||
gen_output = await stage_engine.generate(prompt=prompt, request_id=rid, **diffusion_kwargs)
|
||||
gen_output = await cast(AsyncOmniDiffusion, stage_engine).generate(ein, diffusion_sampling_params, rid)
|
||||
_gen_t1 = _time.time()
|
||||
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
|
||||
await generation_out_q.put((rid, gen_output, _gen_ms))
|
||||
else:
|
||||
# LLM stages: ensure using SamplingParams
|
||||
llm_sampling_params = prepare_sampling_params(sampling_params, "llm")
|
||||
ein = cast(PromptType, ein)
|
||||
llm_sampling_params: SamplingParams = task["sampling_params"]
|
||||
gen_output = None
|
||||
async for res in stage_engine.generate(ein, llm_sampling_params, rid):
|
||||
async for res in cast(AsyncLLM, stage_engine).generate(ein, llm_sampling_params, rid):
|
||||
gen_output = res
|
||||
_gen_t1 = _time.time()
|
||||
_gen_ms = (_gen_t1 - _gen_t0) * 1000.0
|
||||
@@ -1468,7 +1467,7 @@ async def _stage_worker_async(
|
||||
batch_request_ids, batch_request_outputs, _gen_ms_list, batch_metrics
|
||||
):
|
||||
try:
|
||||
r_outputs = [output]
|
||||
r_outputs = [output_strip(output, omni_stage)]
|
||||
use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes)
|
||||
if use_shm:
|
||||
out_q.put(
|
||||
@@ -1554,3 +1553,32 @@ def make_stage_stats(_agg_total_tokens: int, _agg_total_gen_time_ms: float):
|
||||
from vllm_omni.entrypoints.log_utils import StageStats
|
||||
|
||||
return StageStats(total_token=_agg_total_tokens, total_gen_time=_agg_total_gen_time_ms)
|
||||
|
||||
|
||||
def output_strip(r_output: RequestOutput | OmniRequestOutput, omni_stage: OmniStage):
|
||||
"""
|
||||
Strip unnecessary multimodal outputs from stages results,
|
||||
in order to:
|
||||
- reduce memory usage
|
||||
- reduce transfer & serialization overhead
|
||||
"""
|
||||
|
||||
# check multimodal data is required by stage output config.
|
||||
if omni_stage.final_output and omni_stage.final_output_type != "text":
|
||||
return r_output
|
||||
|
||||
# If the request has already finished, should not be altered.
|
||||
if getattr(r_output, "finished", False):
|
||||
return r_output
|
||||
|
||||
mm_output = getattr(r_output, "multimodal_output", None)
|
||||
if mm_output is not None:
|
||||
r_output.multimodal_output = {}
|
||||
|
||||
outputs = getattr(r_output, "outputs", None)
|
||||
if outputs is not None:
|
||||
for out in outputs:
|
||||
if getattr(out, "multimodal_output", None):
|
||||
out.multimodal_output = {}
|
||||
|
||||
return r_output
|
||||
@@ -5,18 +5,20 @@ import multiprocessing.forkserver as forkserver
|
||||
import os
|
||||
|
||||
# Image generation API imports
|
||||
import random
|
||||
import time
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import vllm.envs as envs
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.datastructures import State
|
||||
from starlette.routing import Route
|
||||
from vllm import SamplingParams
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
@@ -79,12 +81,30 @@ from vllm_omni.entrypoints.openai.protocol.images import (
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _remove_route_from_router(
|
||||
router: APIRouter,
|
||||
path: str,
|
||||
methods: set[str] | None = None,
|
||||
) -> None:
|
||||
methods_set = {method.upper() for method in methods} if methods else None
|
||||
for route in list(router.routes):
|
||||
if getattr(route, "path", None) != path:
|
||||
continue
|
||||
if methods_set is not None:
|
||||
route_methods = {method.upper() for method in (getattr(route, "methods", None) or set())}
|
||||
if not (route_methods & methods_set):
|
||||
continue
|
||||
router.routes.remove(route)
|
||||
|
||||
|
||||
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
|
||||
|
||||
|
||||
@@ -741,6 +761,9 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
_remove_route_from_router(router, "/v1/audio/speech", {"POST"})
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/audio/speech",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
@@ -769,6 +792,86 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request
|
||||
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
|
||||
|
||||
|
||||
# Health and Model endpoints for diffusion mode
|
||||
|
||||
|
||||
# Remove existing health endpoint if present (from vllm imports)
|
||||
# to ensure our handler takes precedence
|
||||
_remove_route_from_router(router, "/health")
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health(raw_request: Request) -> JSONResponse:
|
||||
"""Health check endpoint that works for both LLM and diffusion modes.
|
||||
|
||||
Returns 200 OK if the server is healthy.
|
||||
For LLM mode: delegates to engine_client health check
|
||||
For diffusion mode: checks if diffusion_engine is running
|
||||
"""
|
||||
# Check if we're in diffusion mode
|
||||
diffusion_engine = getattr(raw_request.app.state, "diffusion_engine", None)
|
||||
if diffusion_engine is not None:
|
||||
# Diffusion mode health check
|
||||
if hasattr(diffusion_engine, "is_running") and diffusion_engine.is_running:
|
||||
return JSONResponse(content={"status": "healthy"})
|
||||
return JSONResponse(
|
||||
content={"status": "unhealthy", "reason": "Diffusion engine is not running"},
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
)
|
||||
|
||||
# LLM mode - delegate to engine_client
|
||||
engine_client = getattr(raw_request.app.state, "engine_client", None)
|
||||
if engine_client is not None:
|
||||
await engine_client.check_health()
|
||||
return JSONResponse(content={"status": "healthy"})
|
||||
|
||||
return JSONResponse(
|
||||
content={"status": "unhealthy", "reason": "No engine initialized"},
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
)
|
||||
|
||||
|
||||
# Remove existing models endpoint if present (from vllm imports)
|
||||
# to ensure our handler takes precedence
|
||||
_remove_route_from_router(router, "/v1/models")
|
||||
|
||||
|
||||
@router.get("/v1/models")
|
||||
async def show_available_models(raw_request: Request) -> JSONResponse:
|
||||
"""Show available models endpoint that works for both LLM and diffusion modes.
|
||||
|
||||
Returns model information in OpenAI-compatible format.
|
||||
"""
|
||||
# Check if we're in diffusion mode
|
||||
diffusion_model_name = getattr(raw_request.app.state, "diffusion_model_name", None)
|
||||
if diffusion_model_name is not None:
|
||||
# Diffusion mode - return the loaded model
|
||||
return JSONResponse(
|
||||
content={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": diffusion_model_name,
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "vllm-omni",
|
||||
"permission": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
# LLM mode - delegate to openai_serving_models
|
||||
openai_serving_models = getattr(raw_request.app.state, "openai_serving_models", None)
|
||||
if openai_serving_models is not None:
|
||||
models = await openai_serving_models.show_available_models()
|
||||
return JSONResponse(content=models.model_dump())
|
||||
|
||||
return JSONResponse(
|
||||
content={"object": "list", "data": []},
|
||||
)
|
||||
|
||||
|
||||
# Image generation API endpoints
|
||||
|
||||
|
||||
@@ -799,7 +902,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
|
||||
HTTPException: For validation errors, missing engine, or generation failures
|
||||
"""
|
||||
# Get engine client (AsyncOmni) from app state
|
||||
engine_client: EngineClient | None = getattr(raw_request.app.state, "engine_client", None)
|
||||
engine_client: EngineClient | AsyncOmni | None = getattr(raw_request.app.state, "engine_client", None)
|
||||
if engine_client is None or not hasattr(engine_client, "stage_list"):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
@@ -859,10 +962,8 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
|
||||
|
||||
try:
|
||||
# Build params - pass through user values directly
|
||||
gen_params = {
|
||||
"prompt": request.prompt,
|
||||
"num_outputs_per_prompt": request.n,
|
||||
}
|
||||
prompt: OmniTextPrompt = {"prompt": request.prompt}
|
||||
gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n)
|
||||
|
||||
# Parse per-request LoRA (compatible with chat's extra_body.lora shape).
|
||||
if request.lora is not None:
|
||||
@@ -894,62 +995,79 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
|
||||
detail="Invalid lora object: both name and path are required.",
|
||||
)
|
||||
|
||||
gen_params["lora_request"] = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
|
||||
gen_params.lora_request = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
|
||||
if lora_scale is not None:
|
||||
gen_params["lora_scale"] = float(lora_scale)
|
||||
gen_params.lora_scale = float(lora_scale)
|
||||
|
||||
# Parse and add size if provided
|
||||
if request.size:
|
||||
width, height = parse_size(request.size)
|
||||
gen_params["height"] = height
|
||||
gen_params["width"] = width
|
||||
gen_params.height = height
|
||||
gen_params.width = width
|
||||
size_str = f"{width}x{height}"
|
||||
else:
|
||||
size_str = "model default"
|
||||
|
||||
# Add optional parameters ONLY if provided
|
||||
if request.num_inference_steps is not None:
|
||||
gen_params["num_inference_steps"] = request.num_inference_steps
|
||||
gen_params.num_inference_steps = request.num_inference_steps
|
||||
if request.negative_prompt is not None:
|
||||
gen_params["negative_prompt"] = request.negative_prompt
|
||||
prompt["negative_prompt"] = request.negative_prompt
|
||||
if request.guidance_scale is not None:
|
||||
gen_params["guidance_scale"] = request.guidance_scale
|
||||
gen_params.guidance_scale = request.guidance_scale
|
||||
if request.true_cfg_scale is not None:
|
||||
gen_params["true_cfg_scale"] = request.true_cfg_scale
|
||||
gen_params.true_cfg_scale = request.true_cfg_scale
|
||||
if request.seed is not None:
|
||||
gen_params["seed"] = request.seed
|
||||
gen_params["request_id"] = f"img_gen_{int(time.time())}"
|
||||
gen_params.seed = request.seed
|
||||
else:
|
||||
# If seed is not provided, generate a random one to ensure
|
||||
# a proper generator is initialized in the backend.
|
||||
# This fixes issues where using the default global generator
|
||||
# might produce blurry images in some environments.
|
||||
gen_params.seed = random.randint(0, 2**32 - 1)
|
||||
|
||||
request_id = f"img_gen_{int(time.time())}"
|
||||
|
||||
logger.info(f"Generating {request.n} image(s) {size_str}")
|
||||
|
||||
# Generate images using AsyncOmni (multi-stage mode)
|
||||
engine_client = cast(AsyncOmni, engine_client)
|
||||
result = None
|
||||
stage_list = getattr(engine_client, "stage_list", None)
|
||||
if isinstance(stage_list, list):
|
||||
default_params_list = getattr(engine_client, "default_sampling_params_list", None)
|
||||
default_params_list: list[OmniSamplingParams] | None = getattr(
|
||||
engine_client, "default_sampling_params_list", None
|
||||
)
|
||||
if not isinstance(default_params_list, list):
|
||||
default_params_list = [{} for _ in stage_types]
|
||||
default_params_list = [
|
||||
OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types
|
||||
]
|
||||
else:
|
||||
default_params_list = list(default_params_list)
|
||||
if len(default_params_list) != len(stage_types):
|
||||
default_params_list = (default_params_list + [{} for _ in stage_types])[: len(stage_types)]
|
||||
default_params_list = (
|
||||
default_params_list
|
||||
+ [OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types]
|
||||
)[: len(stage_types)]
|
||||
|
||||
sampling_params_list: list[dict[str, Any]] = []
|
||||
sampling_params_list: list[OmniSamplingParams] = []
|
||||
for idx, stage_type in enumerate(stage_types):
|
||||
if stage_type == "diffusion":
|
||||
sampling_params_list.append(gen_params)
|
||||
else:
|
||||
base_params = default_params_list[idx]
|
||||
sampling_params_list.append(dict(base_params) if isinstance(base_params, dict) else base_params)
|
||||
sampling_params_list.append(base_params)
|
||||
|
||||
async for output in engine_client.generate(
|
||||
prompt=gen_params["prompt"],
|
||||
request_id=gen_params["request_id"],
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
):
|
||||
result = output
|
||||
else:
|
||||
result = await engine_client.generate(**gen_params)
|
||||
result = await engine_client.generate(
|
||||
prompt=prompt, request_id=request_id, sampling_params_list=[gen_params]
|
||||
)
|
||||
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
|
||||
@@ -6,13 +6,17 @@ import uuid
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional, cast
|
||||
|
||||
import jinja2
|
||||
import torch
|
||||
from fastapi import Request
|
||||
from PIL import Image
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||
|
||||
try:
|
||||
import soundfile
|
||||
except ImportError:
|
||||
@@ -282,16 +286,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
trace_headers = None if raw_request is None else await self._get_trace_headers(raw_request.headers)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
prompt=engine_prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
output_modalities=output_modalities,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
@@ -1678,10 +1677,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
):
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
final_res = omni_outputs.request_output
|
||||
audio_data = final_res.multimodal_output.get("audio")
|
||||
if stream:
|
||||
audio_tensor = final_res.multimodal_output["audio"][-1].float().detach().cpu().numpy()
|
||||
audio_tensor = audio_data[-1].float().detach().cpu().numpy()
|
||||
else:
|
||||
audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy()
|
||||
if isinstance(audio_data, list):
|
||||
audio_data = torch.cat(audio_data, dim=-1)
|
||||
audio_tensor = audio_data.float().detach().cpu().numpy()
|
||||
|
||||
# Ensure audio is 1D (flatten if needed)
|
||||
if audio_tensor.ndim > 1:
|
||||
@@ -1924,29 +1926,30 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
logger.warning("Failed to decode reference image: %s", e)
|
||||
|
||||
# Build generation kwargs
|
||||
gen_kwargs: dict[str, Any] = {
|
||||
gen_prompt: OmniTextPrompt = {
|
||||
"prompt": prompt,
|
||||
"request_id": request_id,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"negative_prompt": negative_prompt,
|
||||
"num_outputs_per_prompt": num_outputs_per_prompt,
|
||||
"seed": seed,
|
||||
}
|
||||
gen_params = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=num_inference_steps,
|
||||
height=height,
|
||||
width=width,
|
||||
num_outputs_per_prompt=num_outputs_per_prompt,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
if guidance_scale is not None:
|
||||
gen_kwargs["guidance_scale"] = guidance_scale
|
||||
gen_params.guidance_scale = guidance_scale
|
||||
|
||||
# Add Qwen-Image specific parameter
|
||||
if true_cfg_scale is not None:
|
||||
gen_kwargs["true_cfg_scale"] = true_cfg_scale
|
||||
gen_params.true_cfg_scale = true_cfg_scale
|
||||
|
||||
# Add video generation parameters if set
|
||||
if num_frames is not None:
|
||||
gen_kwargs["num_frames"] = num_frames
|
||||
gen_params.num_frames = num_frames
|
||||
if guidance_scale_2 is not None:
|
||||
gen_kwargs["guidance_scale_2"] = guidance_scale_2
|
||||
gen_params.guidance_scale_2 = guidance_scale_2
|
||||
|
||||
# Parse per-request LoRA (works for both AsyncOmniDiffusion and AsyncOmni).
|
||||
if lora_body and isinstance(lora_body, dict):
|
||||
@@ -1969,16 +1972,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
lora_int_id = stable_lora_int_id(str(lora_path))
|
||||
if lora_name and lora_path:
|
||||
lora_req = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
|
||||
gen_kwargs["lora_request"] = lora_req
|
||||
gen_params.lora_request = lora_req
|
||||
if lora_scale is not None:
|
||||
gen_kwargs["lora_scale"] = float(lora_scale)
|
||||
gen_params.lora_scale = float(lora_scale)
|
||||
except Exception as e: # pragma: no cover - safeguard
|
||||
logger.warning("Failed to parse LoRA request: %s", e)
|
||||
|
||||
# Add reference image if provided
|
||||
if pil_images:
|
||||
if len(pil_images) == 1:
|
||||
gen_kwargs["pil_image"] = pil_images[0]
|
||||
gen_prompt["multi_modal_data"] = {}
|
||||
gen_prompt["multi_modal_data"]["image"] = pil_images[0]
|
||||
else:
|
||||
od_config = getattr(self._diffusion_engine, "od_config", None)
|
||||
supports_multimodal_inputs = getattr(od_config, "supports_multimodal_inputs", False)
|
||||
@@ -1986,7 +1990,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
# TODO: entry is asyncOmni. We hack the od config here.
|
||||
supports_multimodal_inputs = True
|
||||
if supports_multimodal_inputs:
|
||||
gen_kwargs["pil_image"] = pil_images
|
||||
gen_prompt["multi_modal_data"] = {}
|
||||
gen_prompt["multi_modal_data"]["image"] = pil_images
|
||||
else:
|
||||
return self._create_error_response(
|
||||
"Multiple input images are not supported by the current diffusion model. "
|
||||
@@ -1999,18 +2004,24 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
# Handle both AsyncOmniDiffusion (returns OmniRequestOutput) and AsyncOmni (returns AsyncGenerator)
|
||||
if hasattr(self._diffusion_engine, "stage_list"):
|
||||
# AsyncOmni: iterate through async generator to get final output
|
||||
diffusion_engine = cast(AsyncOmni, self._diffusion_engine)
|
||||
result = None
|
||||
async for output in self._diffusion_engine.generate(
|
||||
prompt=gen_kwargs["prompt"],
|
||||
request_id=gen_kwargs.get("request_id"),
|
||||
sampling_params_list=[gen_kwargs], # Pass as single-stage params
|
||||
async for output in diffusion_engine.generate(
|
||||
prompt=gen_prompt,
|
||||
sampling_params_list=[gen_params], # Pass as single-stage params
|
||||
request_id=request_id,
|
||||
):
|
||||
result = output
|
||||
if result is None:
|
||||
return self._create_error_response("No output generated from AsyncOmni")
|
||||
else:
|
||||
# AsyncOmniDiffusion: direct call
|
||||
result = await self._diffusion_engine.generate(**gen_kwargs)
|
||||
diffusion_engine = cast(AsyncOmniDiffusion, self._diffusion_engine)
|
||||
result = await diffusion_engine.generate(
|
||||
prompt=gen_prompt,
|
||||
sampling_params=gen_params,
|
||||
request_id=request_id,
|
||||
)
|
||||
# Extract images from result
|
||||
# Handle nested OmniRequestOutput structure where images might be in request_output
|
||||
images = getattr(result.request_output, "images", [])
|
||||
|
||||
@@ -57,12 +57,12 @@ def set_stage_devices(
|
||||
- CUDA: Sets CUDA_VISIBLE_DEVICES and calls torch.cuda.set_device()
|
||||
- NPU: Sets ASCEND_RT_VISIBLE_DEVICES and calls torch.npu.set_device()
|
||||
"""
|
||||
from vllm_omni.utils import detect_device_type, get_device_control_env_var
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if device_type is None:
|
||||
device_type = detect_device_type()
|
||||
device_type = current_omni_platform.device_type
|
||||
|
||||
env_var = get_device_control_env_var()
|
||||
env_var = current_omni_platform.device_control_env_var
|
||||
|
||||
try:
|
||||
selected_physical: int | None = None
|
||||
|
||||
@@ -10,7 +10,7 @@ from vllm.transformers_utils.config import get_config, get_hf_file_to_dict
|
||||
from vllm.transformers_utils.repo_utils import file_or_path_exists
|
||||
|
||||
from vllm_omni.entrypoints.stage_utils import _to_dict
|
||||
from vllm_omni.utils import detect_device_type, is_rocm
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Get the project root directory (2 levels up from this file)
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||
@@ -164,16 +164,12 @@ def resolve_model_config_path(model: str) -> str:
|
||||
f"Model is not in standard transformers format and does not have model_index.json. "
|
||||
f"Please ensure the model has proper configuration files with 'model_type' field"
|
||||
)
|
||||
device_type = detect_device_type()
|
||||
|
||||
# Try device-specific config first
|
||||
if device_type != "cuda" or is_rocm():
|
||||
device_config_file = f"vllm_omni/model_executor/stage_configs/{device_type}/{model_type}.yaml"
|
||||
if is_rocm():
|
||||
device_config_file = f"vllm_omni/model_executor/stage_configs/rocm/{model_type}.yaml"
|
||||
device_config_path = PROJECT_ROOT / device_config_file
|
||||
if os.path.exists(device_config_path):
|
||||
return str(device_config_path)
|
||||
default_config_path = current_omni_platform.get_default_stage_config_path()
|
||||
model_type_str = f"{model_type}.yaml"
|
||||
complete_config_path = PROJECT_ROOT / default_config_path / model_type_str
|
||||
if os.path.exists(complete_config_path):
|
||||
return str(complete_config_path)
|
||||
|
||||
# Fall back to default config
|
||||
stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml"
|
||||
|
||||
Reference in new issue
Block a user