Merge origin/main into dev/vllm-align; resolve omni_stage.py conflicts

Co-authored-by: Cursor <cursoragent@cursor.com>

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouamandCursor committed 2026-02-09 01:52:31 +00:00
commit 03c6a58f69
18 files changed
+761 -381

No files matched your search

+1 -1
View File
@@ -245,7 +245,7 @@ class OmniServeCommand(CLISubcommand):
# Diffusion model mixed precision
omni_config_group.add_argument(
"--max-generated-image-size",
type=float,
type=int,
help="The max size of generate image (height * width).",
)
return serve_parser
+179 -343
View File
@@ -16,6 +16,7 @@ import sys
import time
import traceback
from collections.abc import Sequence
from contextlib import contextmanager
from dataclasses import fields
from typing import Any, Literal, cast
@@ -55,6 +56,144 @@ from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
@contextmanager
def _sequential_init_lock(engine_args: dict[str, Any], stage_init_timeout: int = 300):
"""Acquire device locks for sequential init if NVML is unavailable.
If process-scoped memory tracking is available (NVML works), stages can
safely initialize concurrently — each measures only its own GPU memory.
Otherwise, fall back to file-based locks to serialize initialization.
"""
from vllm_omni.worker.gpu_memory_utils import is_process_scoped_memory_available
if is_process_scoped_memory_available():
logger.debug("NVML process-scoped memory available — concurrent init is safe, skipping locks")
yield
return
logger.debug("NVML unavailable — using sequential init locks")
from vllm_omni.platforms import current_omni_platform
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1
cfg_parallel_size = 1
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Get physical device IDs from device control env var
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = os.environ.get(device_control_env)
physical_devices = []
if visible_devices_str:
try:
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
except (ValueError, IndexError):
pass
if not physical_devices:
num_devices = current_omni_platform.get_device_count()
physical_devices = list(range(num_devices))
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = sorted(physical_devices[:num_devices_to_lock])
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
cfg_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
wait_start = time.time()
acquired_lock_fds = []
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
lock_fd = os.open(lock_file, os.O_CREAT | os.O_RDWR, 0o644)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
os.ftruncate(lock_fd, 0)
os.write(lock_fd, f"{os.getpid()}\n".encode())
os.fsync(lock_fd)
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
os.close(lock_fd)
if time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway",
device_id,
)
break
time.sleep(0.1)
except OSError as e:
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
device_id,
e,
)
try:
os.close(lock_fd)
except (OSError, NameError):
pass
break
# Set FD_CLOEXEC to prevent child processes from inheriting locks
for lock_fd in acquired_lock_fds:
try:
flags = fcntl.fcntl(lock_fd, fcntl.F_GETFD)
fcntl.fcntl(lock_fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
except (OSError, ValueError):
pass
try:
yield
finally:
for lock_fd in acquired_lock_fds:
try:
fcntl.flock(lock_fd, fcntl.LOCK_UN)
os.close(lock_fd)
logger.debug("Released initialization lock (fd=%s)", lock_fd)
except (OSError, ValueError):
pass
def _resolve_worker_cls(engine_args: dict[str, Any]) -> None:
worker_type = engine_args.pop("worker_type", None)
if not worker_type:
@@ -541,183 +680,32 @@ def _stage_worker(
except Exception as e:
logger.warning("Device setup failed: %s", e)
# Sequential initialization on the same device to avoid memory calculation errors
# when multiple instances start simultaneously
# For TP/PP/DP/SP, we need to lock ALL devices that will be used by this stage
lock_files = []
try:
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
# Use sequential init locks only when NVML is unavailable
with _sequential_init_lock(engine_args, stage_init_timeout):
# Init engine based on stage_type
logger.debug(
"[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys())
)
if engine_args.get("async_chunk", False):
logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id)
stage_connector_spec = {}
for v in connectors_config.values():
stage_connector_spec = dict(v.get("spec", {}))
break
engine_args["stage_connector_spec"] = stage_connector_spec
engine_args["stage_id"] = stage_id
if stage_type == "diffusion":
engine_args.pop("model_stage", None)
engine_args.pop("model", None)
stage_engine = OmniDiffusion(
model=model,
stage_id=stage_id,
engine_input_source=stage_payload.get("engine_input_source", []),
**engine_args,
)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1 # not use in omni model
cfg_parallel_size = 1 # not used in omni model
# Calculate total number of devices needed for this stage
# For a single stage worker:
# - TP: splits model across devices (always needed)
# - PP: splits layers across pipeline stages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# - SP: sequence parallelism, typically uses TP devices
# - CFG: Classifier-Free Guidance parallelism for diffusion models
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
# After set_stage_devices, this env var is set to physical device(s)
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = _os.environ.get(device_control_env)
physical_devices = []
if visible_devices_str:
try:
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
except (ValueError, IndexError):
pass
if not physical_devices:
# Fallback: use logical device count if device control env var not set
num_devices = current_omni_platform.get_device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
cfg_parallel_size,
num_devices_to_lock,
devices_to_lock,
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway",
device_id,
)
break
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
device_id,
e,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
# Set FD_CLOEXEC on all lock file descriptors to prevent child processes
# (e.g., EngineCore) from inheriting them, which would cause deadlock
for lock_fd in acquired_lock_fds:
try:
flags = fcntl.fcntl(lock_fd, fcntl.F_GETFD)
fcntl.fcntl(lock_fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
except (OSError, ValueError):
pass
except Exception as e:
logger.debug(
"[Stage-%s] Failed to set up sequential initialization lock: %s",
stage_id,
e,
)
# Init engine based on stage_type
logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys()))
if engine_args.get("async_chunk", False):
logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id)
stage_connector_spec = {}
for v in connectors_config.values():
stage_connector_spec = dict(v.get("spec", {}))
break
engine_args["stage_connector_spec"] = stage_connector_spec
engine_args["stage_id"] = stage_id
if stage_type == "diffusion":
engine_args.pop("model_stage", None)
engine_args.pop("model", None)
stage_engine = OmniDiffusion(
model=model,
stage_id=stage_id,
engine_input_source=stage_payload.get("engine_input_source", []),
**engine_args,
)
else:
# Default to LLM engine
stage_engine = OmniLLM(model=model, **engine_args)
# Release all locks AFTER engine initialization completes
for lock_fd in lock_files:
try:
fcntl.flock(lock_fd, fcntl.LOCK_UN)
_os.close(lock_fd)
logger.debug("Released initialization lock (fd=%s)", lock_fd)
except (OSError, ValueError):
pass
lock_files = [] # Clear after release
# Default to LLM engine
stage_engine = OmniLLM(model=model, **engine_args)
logger.debug("Engine initialized")
# Initialize OmniConnectors if configured
@@ -1086,156 +1074,23 @@ async def _stage_worker_async(
return
connectors = built_connectors
# Sequential initialization on the same device to avoid memory calculation errors
# when multiple instances start simultaneously
# For TP/PP/DP/PCP, we need to lock ALL devices that will be used by this stage
lock_files = []
try:
# Get all parallel sizes from engine_args or parallel_config (defaults to 1)
if "parallel_config" in engine_args:
parallel_config = engine_args["parallel_config"]
tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1)
pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1)
data_parallel_size = parallel_config.get("data_parallel_size", 1)
prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1)
sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1)
cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1)
else:
tensor_parallel_size = engine_args.get("tensor_parallel_size", 1)
pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1)
data_parallel_size = engine_args.get("data_parallel_size", 1)
prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1)
sequence_parallel_size = 1 # not use in omni model
cfg_parallel_size = 1 # not used in omni model
# Calculate total number of devices needed for this stage
# For a single stage worker:
# - TP: splits model across devices (always needed)
# - PP: splits layers across pipeline stages, but each stage uses TP devices
# - DP: replicates model, but each replica uses TP devices
# - PCP: context parallelism, typically uses TP devices
# - SP: sequence parallelism, typically uses TP devices
# - CFG: Classifier-Free Guidance parallelism for diffusion models
# The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size
# (PP/DP/PCP are higher-level parallelism that don't add devices per stage)
num_devices_per_stage = (
tensor_parallel_size
* pipeline_parallel_size
* data_parallel_size
* prefill_context_parallel_size
* sequence_parallel_size
* cfg_parallel_size
)
# Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES)
# After set_stage_devices, this env var is set to physical device(s)
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = _os.environ.get(device_control_env)
physical_devices = []
if visible_devices_str:
try:
physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()]
except (ValueError, IndexError):
pass
if not physical_devices:
# Fallback: use logical device count if device control env var not set
num_devices = current_omni_platform.get_device_count()
physical_devices = list(range(num_devices))
# Determine which devices will be used (min of devices per stage and available devices)
num_devices_to_lock = min(num_devices_per_stage, len(physical_devices))
devices_to_lock = physical_devices[:num_devices_to_lock]
# Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order)
devices_to_lock = sorted(devices_to_lock)
# Use sequential init locks only when NVML is unavailable
with _sequential_init_lock(engine_args, stage_init_timeout):
# Init engine based on stage_type
logger.debug(
"Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s",
tensor_parallel_size,
pipeline_parallel_size,
data_parallel_size,
prefill_context_parallel_size,
sequence_parallel_size,
cfg_parallel_size,
num_devices_to_lock,
devices_to_lock,
"[Stage-%s] Initializing %s engine with args keys=%s",
stage_id,
stage_type,
list(engine_args.keys()),
)
# Acquire exclusive locks for all devices using fcntl.flock
# Locks are automatically released when process dies
wait_start = _time.time()
acquired_lock_fds = [] # Store file descriptors to keep locks alive
for device_id in devices_to_lock:
lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock"
lock_acquired = False
while not lock_acquired:
try:
# Open or create the lock file
lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644)
# Try to acquire exclusive lock (non-blocking first)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
# Successfully acquired lock - write PID
_os.ftruncate(lock_fd, 0) # Clear file
_os.write(lock_fd, f"{_os.getpid()}\n".encode())
_os.fsync(lock_fd) # Ensure written to disk
lock_acquired = True
acquired_lock_fds.append(lock_fd)
logger.debug("Acquired exclusive lock for device %s", device_id)
except BlockingIOError:
# Lock is held by another process
_os.close(lock_fd)
# Check if we've been waiting too long
if _time.time() - wait_start > stage_init_timeout:
logger.warning(
"Timeout waiting for device %s initialization lock, proceeding anyway with timeout %s",
device_id,
stage_init_timeout,
)
break
# Wait a bit before retrying
_time.sleep(0.1)
except OSError as e:
# Other error - log and continue without lock
logger.debug(
"Failed to acquire lock for device %s: %s, continuing anyway",
device_id,
e,
)
try:
_os.close(lock_fd)
except (OSError, NameError):
pass
break
lock_files = acquired_lock_fds
except Exception as e:
logger.debug("Failed to set up sequential initialization lock: %s", e)
# Init engine based on stage_type
logger.debug(
"[Stage-%s] Initializing %s engine with args keys=%s",
stage_id,
stage_type,
list(engine_args.keys()),
)
if engine_args.get("async_chunk", False):
logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id)
stage_connector_spec = {}
for v in connectors_config.values():
stage_connector_spec = dict(v.get("spec", {}))
break
engine_args["stage_connector_spec"] = stage_connector_spec
engine_args["stage_id"] = stage_id
stage_engine = None
engine_init_error = None
try:
if engine_args.get("async_chunk", False):
logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id)
stage_connector_spec = {}
for v in connectors_config.values():
stage_connector_spec = dict(v.get("spec", {}))
break
engine_args["stage_connector_spec"] = stage_connector_spec
engine_args["stage_id"] = stage_id
if stage_type == "diffusion":
# For diffusion, we need to extract diffusion-specific config
od_config = _build_od_config(engine_args, model)
@@ -1262,25 +1117,6 @@ async def _stage_worker_async(
usage_context=usage_context,
engine_args=omni_engine_args,
)
except Exception as e:
engine_init_error = e
logger.exception("[Stage-%s] Engine initialization failed: %s", stage_id, e)
finally:
# Release all locks by closing file descriptors
# Locks are automatically released when file descriptors are closed
# or when process dies
for lock_fd in lock_files:
try:
fcntl.flock(lock_fd, fcntl.LOCK_UN)
_os.close(lock_fd)
logger.debug("Released initialization lock (fd=%s)", lock_fd)
except (OSError, ValueError):
pass
# If engine init failed, re-raise the exception
if engine_init_error is not None:
raise engine_init_error
if stage_engine is None:
raise RuntimeError(f"[Stage-{stage_id}] Engine was not initialized")
omni_stage.set_async_engine(stage_engine)
if hasattr(omni_stage.async_engine, "log_stats") and omni_stage.async_engine.log_stats:
+4 -2
View File
@@ -970,7 +970,9 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
# a proper generator is initialized in the backend.
# This fixes issues where using the default global generator
# might produce blurry images in some environments.
_update_if_not_none(gen_params, "seed", random.randint(0, 2**32 - 1) if request.seed is None else request.seed)
_update_if_not_none(
gen_params, "seed", request.seed if request.seed is not None else random.randint(0, 2**32 - 1)
)
request_id = f"img_gen_{uuid.uuid4().hex}"
@@ -1128,7 +1130,7 @@ async def edit_images(
# a proper generator is initialized in the backend.
# This fixes issues where using the default global generator
# might produce blurry images in some environments.
_update_if_not_none(gen_params, "seed", seed or random.randint(0, 2**32 - 1))
_update_if_not_none(gen_params, "seed", seed if seed is not None else random.randint(0, 2**32 - 1))
# 4. Generate images using AsyncOmni (multi-stage mode)
request_id = f"img_edit_{int(time.time())}"