diff --git a/tests/entrypoints/test_async_omni_diffusion_config.py b/tests/entrypoints/test_async_omni_diffusion_config.py index ce443d5a5..ed205032e 100644 --- a/tests/entrypoints/test_async_omni_diffusion_config.py +++ b/tests/entrypoints/test_async_omni_diffusion_config.py @@ -11,12 +11,18 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] MODEL = "riverclouds/qwen_image_random" +def _noop_inline_engine(self, model, stage_config, kwargs): + self._inline_diffusion = False + self._inline_engine = None + + def test_default_stage_config_includes_cache_backend(monkeypatch): """Ensure cache_backend/cache_config are preserved in default diffusion stage.""" monkeypatch.setattr(utils_module, "load_stage_configs_from_model", lambda model, base_engine_args=None: []) monkeypatch.setattr(utils_module, "resolve_model_config_path", lambda model: None) monkeypatch.setattr(AsyncOmni, "_start_stages", lambda self, model: None) monkeypatch.setattr(AsyncOmni, "_wait_for_stages_ready", lambda self, timeout=0: None) + monkeypatch.setattr(AsyncOmni, "_init_inline_diffusion_engine", _noop_inline_engine) omni = AsyncOmni( model=MODEL, @@ -47,6 +53,7 @@ def test_default_cache_config_used_when_missing(monkeypatch): monkeypatch.setattr(utils_module, "resolve_model_config_path", lambda model: None) monkeypatch.setattr(AsyncOmni, "_start_stages", lambda self, model: None) monkeypatch.setattr(AsyncOmni, "_wait_for_stages_ready", lambda self, timeout=0: None) + monkeypatch.setattr(AsyncOmni, "_init_inline_diffusion_engine", _noop_inline_engine) omni = AsyncOmni( model=MODEL, @@ -65,6 +72,7 @@ def test_default_stage_devices_from_sequence_parallel(monkeypatch): monkeypatch.setattr(utils_module, "resolve_model_config_path", lambda model: None) monkeypatch.setattr(AsyncOmni, "_start_stages", lambda self, model: None) monkeypatch.setattr(AsyncOmni, "_wait_for_stages_ready", lambda self, timeout=0: None) + monkeypatch.setattr(AsyncOmni, "_init_inline_diffusion_engine", _noop_inline_engine) omni = AsyncOmni( model=MODEL, diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 103f0ca89..abd668466 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -67,14 +67,20 @@ class DiffusionEngine: raise e def step(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: + diffusion_engine_start_time = time.perf_counter() + # Apply pre-processing if available + preprocess_time = 0.0 if self.pre_process_func is not None: - preprocess_start_time = time.time() + preprocess_start_time = time.perf_counter() request = self.pre_process_func(request) - preprocess_time = time.time() - preprocess_start_time + preprocess_time = time.perf_counter() - preprocess_start_time logger.info(f"Pre-processing completed in {preprocess_time:.4f} seconds") + exec_start_time = time.perf_counter() output = self.add_req_and_wait_for_response(request) + exec_total_time = time.perf_counter() - exec_start_time + if output.error: raise Exception(f"{output.error}") logger.info("Generation completed successfully.") @@ -92,21 +98,34 @@ class DiffusionEngine: for i, prompt in enumerate(request.prompts) ] - postprocess_start_time = time.time() + postprocess_start_time = time.perf_counter() outputs = self.post_process_func(output.output) if self.post_process_func is not None else output.output audio_payload = None if isinstance(outputs, dict): audio_payload = outputs.get("audio") outputs = outputs.get("video", outputs) - postprocess_time = time.time() - postprocess_start_time + postprocess_time = time.perf_counter() - postprocess_start_time logger.info(f"Post-processing completed in {postprocess_time:.4f} seconds") + step_total_ms = (time.perf_counter() - diffusion_engine_start_time) * 1000 + logger.info( + "DiffusionEngine.step breakdown: preprocess=%.2f ms, " + "add_req_and_wait=%.2f ms, postprocess=%.2f ms, total=%.2f ms", + preprocess_time * 1000, + exec_total_time * 1000, + postprocess_time * 1000, + step_total_ms, + ) + # Convert to OmniRequestOutput format # Ensure outputs is a list if not isinstance(outputs, list): outputs = [outputs] if outputs is not None else [] metrics = { + "preprocess_time_ms": preprocess_time * 1000, + "diffusion_engine_exec_time_ms": (time.perf_counter() - diffusion_engine_start_time) * 1000, + "diffusion_engine_total_time_ms": exec_total_time * 1000, "image_num": int(request.sampling_params.num_outputs_per_prompt), "resolution": int(request.sampling_params.resolution), "postprocess_time_ms": postprocess_time * 1000, diff --git a/vllm_omni/diffusion/ipc.py b/vllm_omni/diffusion/ipc.py new file mode 100644 index 000000000..cac406ef4 --- /dev/null +++ b/vllm_omni/diffusion/ipc.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""IPC utilities for transferring large tensors via POSIX shared memory. + +Used by Hop1 (GPU worker <-> scheduler) to avoid pickling large video tensors +through the MessageQueue. Tensors above ``_SHM_TENSOR_THRESHOLD`` are copied +into a named shared-memory segment; only a lightweight metadata dict is +serialised through the queue. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from vllm_omni.diffusion.data import DiffusionOutput + +_SHM_TENSOR_THRESHOLD = 1_000_000 # 1 MB + + +def _tensor_to_shm(tensor: torch.Tensor) -> dict[str, Any]: + """Copy a tensor into POSIX shared memory and return a metadata handle. + + The shared memory segment remains alive after this call (the local fd is + closed, but the segment persists until ``_tensor_from_shm`` unlinks it). + """ + from multiprocessing import shared_memory + + import numpy as np + + tensor = tensor.detach().cpu().contiguous() + arr = tensor.numpy() + nbytes = arr.nbytes + shm = shared_memory.SharedMemory(create=True, size=nbytes) + shm_arr = np.ndarray(arr.shape, dtype=arr.dtype, buffer=shm.buf[:nbytes]) + np.copyto(shm_arr, arr) + handle = { + "__tensor_shm__": True, + "name": shm.name, + "shape": list(tensor.shape), + "torch_dtype": str(tensor.dtype), + "numpy_dtype": str(arr.dtype), + "nbytes": nbytes, + } + shm.close() + return handle + + +def _tensor_from_shm(handle: dict[str, Any]) -> torch.Tensor: + """Reconstruct a tensor from a shared-memory handle and free the segment.""" + from multiprocessing import shared_memory + + import numpy as np + + shm = shared_memory.SharedMemory(name=handle["name"]) + try: + np_dtype = np.dtype(handle["numpy_dtype"]) + arr = np.ndarray(handle["shape"], dtype=np_dtype, buffer=shm.buf[: handle["nbytes"]]) + tensor = torch.from_numpy(arr.copy()) + finally: + shm.close() + shm.unlink() + return tensor + + +def pack_diffusion_output_shm(output: DiffusionOutput) -> DiffusionOutput: + """Replace large tensors in *output* with shared-memory handles. + + The DiffusionOutput is modified **in-place** so that the (now lightweight) + object can be serialised cheaply through a MessageQueue. + """ + if output.output is not None and isinstance(output.output, torch.Tensor): + if output.output.nelement() * output.output.element_size() > _SHM_TENSOR_THRESHOLD: + output.output = _tensor_to_shm(output.output) + if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor): + if output.trajectory_latents.nelement() * output.trajectory_latents.element_size() > _SHM_TENSOR_THRESHOLD: + output.trajectory_latents = _tensor_to_shm(output.trajectory_latents) + return output + + +def unpack_diffusion_output_shm(output: DiffusionOutput) -> DiffusionOutput: + """Reconstruct tensors from shared-memory handles produced by ``pack_diffusion_output_shm``.""" + if isinstance(output.output, dict) and output.output.get("__tensor_shm__"): + output.output = _tensor_from_shm(output.output) + if isinstance(output.trajectory_latents, dict) and output.trajectory_latents.get("__tensor_shm__"): + output.trajectory_latents = _tensor_from_shm(output.trajectory_latents) + return output diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index 1de801c10..9a0037a6a 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -6,6 +6,7 @@ from __future__ import annotations import json import logging import os +import time from collections.abc import Iterable from typing import Any, cast @@ -21,7 +22,7 @@ from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import Dist from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader -from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -29,6 +30,7 @@ from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) +DEBUG_PERF = False def retrieve_latents( @@ -424,7 +426,11 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): if generator is None and req.sampling_params.seed is not None: generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) - # Encode prompts + if DEBUG_PERF: + # Sync GPU before timing to ensure accurate measurements + current_omni_platform.synchronize() + _t_pipeline_start = time.perf_counter() + _t_text_enc_start = _t_pipeline_start if prompt_embeds is None: prompt_embeds, negative_prompt_embeds = self.encode_prompt( prompt=prompt, @@ -443,6 +449,9 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): raise ValueError( "negative_prompt_embeds must be provided when prompt_embeds are given and guidance > 1." ) + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_text_enc_ms = (time.perf_counter() - _t_text_enc_start) * 1000 # Timesteps self.scheduler.set_timesteps(num_steps, device=device) @@ -452,7 +461,8 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): if boundary_ratio is not None: boundary_timestep = boundary_ratio * self.scheduler.config.num_train_timesteps - # Handle I2V mode when expand_timesteps=True and image is provided + if DEBUG_PERF: + _t_latent_prep_start = time.perf_counter() multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None if isinstance(raw_image, list): @@ -543,11 +553,15 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): generator=generator, latents=req.sampling_params.latents, ) + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_latent_prep_ms = (time.perf_counter() - _t_latent_prep_start) * 1000 if attention_kwargs is None: attention_kwargs = {} - # Denoising + if DEBUG_PERF: + _t_denoise_start = time.perf_counter() with self.progress_bar(total=len(timesteps)) as pbar: for t in timesteps: self._current_timestep = t @@ -638,12 +652,16 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): if current_omni_platform.is_available(): current_omni_platform.empty_cache() self._current_timestep = None + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_denoise_ms = (time.perf_counter() - _t_denoise_start) * 1000 # For I2V mode: blend final latents with condition if self.expand_timesteps and latent_condition is not None: latents = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents - # Decode + if DEBUG_PERF: + _t_decode_start = time.perf_counter() if output_type == "latent": output = latents else: @@ -659,6 +677,28 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin): latents = latents / latents_std + latents_mean output = self.vae.decode(latents, return_dict=False)[0] + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_decode_ms = (time.perf_counter() - _t_decode_start) * 1000 + _t_pipeline_wall_ms = (time.perf_counter() - _t_pipeline_start) * 1000 + _t_stages_sum = _t_text_enc_ms + _t_latent_prep_ms + _t_denoise_ms + _t_decode_ms + + if _is_rank_zero(): + logger.info( + "Pipeline stage timing summary: " + "TextEncoding=%.2f ms, LatentPreparation=%.2f ms, " + "Denoising=%.2f ms (%d steps), Decoding=%.2f ms, " + "StagesSum=%.2f ms, PipelineWall=%.2f ms, Unaccounted=%.2f ms", + _t_text_enc_ms, + _t_latent_prep_ms, + _t_denoise_ms, + len(timesteps), + _t_decode_ms, + _t_stages_sum, + _t_pipeline_wall_ms, + _t_pipeline_wall_ms - _t_stages_sum, + ) + return DiffusionOutput(output=output) def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor: diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index b673cd0fa..fe15a24f5 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging import os +import time from collections.abc import Iterable from typing import Any, cast @@ -22,7 +23,7 @@ from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput -from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin +from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( create_transformer_from_config, @@ -34,6 +35,7 @@ from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) +DEBUG_PERF = False def _load_model_index(model: str, local_files_only: bool) -> dict: @@ -388,7 +390,12 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB if generator is None and req.sampling_params.seed is not None: generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) - # Encode prompts + if DEBUG_PERF: + # Sync GPU before timing to ensure accurate measurements + current_omni_platform.synchronize() + _t_pipeline_start = time.perf_counter() + _t_text_enc_start = _t_pipeline_start + if prompt_embeds is None: prompt_embeds, negative_prompt_embeds = self.encode_prompt( prompt=prompt, @@ -404,9 +411,14 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB if negative_prompt_embeds is not None: negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype) + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_text_enc_ms = (time.perf_counter() - _t_text_enc_start) * 1000 + batch_size = prompt_embeds.shape[0] - # Encode image embeddings (for Wan2.1-style with CLIP) + if DEBUG_PERF: + _t_img_enc_start = time.perf_counter() if self.has_image_encoder and self.transformer.config.image_dim is not None: if image_embeds is None: if last_image is None: @@ -418,6 +430,10 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB else: image_embeds = None + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_img_enc_ms = (time.perf_counter() - _t_img_enc_start) * 1000 + # Timesteps self.scheduler.set_timesteps(num_steps, device=device) timesteps = self.scheduler.timesteps @@ -430,7 +446,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB # Prepare latents (use out_channels=16 for VAE latent, not in_channels=36) num_channels_latents = self.transformer.config.out_channels - # Preprocess image for VAE + if DEBUG_PERF: + _t_latent_prep_start = time.perf_counter() from diffusers.video_processor import VideoProcessor video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) @@ -465,10 +482,15 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB last_image=last_image_tensor, ) + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_latent_prep_ms = (time.perf_counter() - _t_latent_prep_start) * 1000 + if attention_kwargs is None: attention_kwargs = {} - # Denoising loop + if DEBUG_PERF: + _t_denoise_start = time.perf_counter() with self.progress_bar(total=len(timesteps)) as pbar: for t in timesteps: self._current_timestep = t @@ -538,11 +560,17 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB current_omni_platform.empty_cache() self._current_timestep = None + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_denoise_ms = (time.perf_counter() - _t_denoise_start) * 1000 + # For expand_timesteps mode, blend final latents with condition if self.expand_timesteps: latents = (1 - first_frame_mask) * condition + first_frame_mask * latents - # Decode + if DEBUG_PERF: + _t_decode_start = time.perf_counter() + if output_type == "latent": output = latents else: @@ -558,6 +586,30 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, ProgressB latents = latents / latents_std + latents_mean output = self.vae.decode(latents, return_dict=False)[0] + if DEBUG_PERF: + current_omni_platform.synchronize() + _t_decode_ms = (time.perf_counter() - _t_decode_start) * 1000 + _t_pipeline_wall_ms = (time.perf_counter() - _t_pipeline_start) * 1000 + _t_stages_sum = _t_text_enc_ms + _t_img_enc_ms + _t_latent_prep_ms + _t_denoise_ms + _t_decode_ms + + if _is_rank_zero(): + logger.info( + "Pipeline stage timing summary: " + "TextEncoding=%.2f ms, ImageEncoding=%.2f ms, " + "LatentPreparation=%.2f ms, Denoising=%.2f ms (%d steps), " + "Decoding=%.2f ms, StagesSum=%.2f ms, PipelineWall=%.2f ms, " + "Unaccounted=%.2f ms", + _t_text_enc_ms, + _t_img_enc_ms, + _t_latent_prep_ms, + _t_denoise_ms, + len(timesteps), + _t_decode_ms, + _t_stages_sum, + _t_pipeline_wall_ms, + _t_pipeline_wall_ms - _t_stages_sum, + ) + return DiffusionOutput(output=output) def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor: diff --git a/vllm_omni/diffusion/scheduler.py b/vllm_omni/diffusion/scheduler.py index c30d907aa..5f1d01c12 100644 --- a/vllm_omni/diffusion/scheduler.py +++ b/vllm_omni/diffusion/scheduler.py @@ -8,6 +8,7 @@ from vllm.distributed.device_communicators.shm_broadcast import MessageQueue from vllm.logger import init_logger from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.ipc import unpack_diffusion_output_shm from vllm_omni.diffusion.request import OmniDiffusionRequest logger = init_logger(__name__) @@ -59,12 +60,18 @@ class Scheduler: # Broadcast RPC request to all workers self.mq.enqueue(rpc_request) - # Wait for result from Rank 0 (or whoever sends it) + # Wait for result from Rank 0 (or whoever sends it) if self.result_mq is None: raise RuntimeError("Result queue not initialized") output = self.result_mq.dequeue() + + try: + unpack_diffusion_output_shm(output) + except Exception as e: + logger.warning("SHM unpack failed (data may already be inline): %s", e) + # {"status": "error", "error": str(e)} if isinstance(output, dict) and output.get("status") == "error": raise RuntimeError("worker error") diff --git a/vllm_omni/diffusion/worker/diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py index 2f073b368..19ba5f31c 100644 --- a/vllm_omni/diffusion/worker/diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -34,6 +34,7 @@ from vllm_omni.diffusion.distributed.parallel_state import ( initialize_model_parallel, ) from vllm_omni.diffusion.forward_context import set_forward_context +from vllm_omni.diffusion.ipc import pack_diffusion_output_shm from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager from vllm_omni.diffusion.profiler import CurrentProfiler from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -376,6 +377,10 @@ class WorkerProc: def return_result(self, output: DiffusionOutput): """Reply to client, only on rank 0.""" if self.result_mq is not None: + try: + pack_diffusion_output_shm(output) + except Exception as e: + logger.warning("SHM pack failed, falling back to raw enqueue: %s", e) self.result_mq.enqueue(output) def recv_message(self): diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 46bbd0c19..6a4b88dbf 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -40,8 +40,15 @@ _R = TypeVar("_R") logger = init_logger(__name__) -def _weak_close_cleanup_async(stage_list, stage_in_queues, stage_out_queues, ray_pg, output_handler, zmq_ctx=None): +def _weak_close_cleanup_async( + stage_list, stage_in_queues, stage_out_queues, ray_pg, output_handler, zmq_ctx=None, inline_engine=None +): """Weak reference cleanup function for AsyncOmni instances.""" + if inline_engine is not None: + try: + inline_engine.close() + except Exception as e: + logger.warning("Failed to close inline diffusion engine: %s", e) if stage_list: for q in stage_in_queues: try: @@ -130,6 +137,7 @@ class AsyncOmni(OmniBase): self._ray_pg, self.output_handler, self._zmq_ctx, + getattr(self, "_inline_engine", None), ) def _create_default_diffusion_stage_cfg(self, kwargs: dict[str, Any]) -> dict[str, Any]: @@ -330,6 +338,11 @@ class AsyncOmni(OmniBase): async with self._pause_cond: await self._pause_cond.wait_for(lambda: not self._paused) + if self._inline_diffusion: + async for output in self._generate_inline(prompt, request_id, sampling_params_list, output_modalities): + yield output + return + logger.debug(f"[{self._name}] generate() called") try: # Start output handler on the first call to generate() @@ -421,6 +434,97 @@ class AsyncOmni(OmniBase): logger.info("[AsyncOrchestrator] Request %s aborted.", request_id) raise + async def _generate_inline( + self, + prompt: OmniPromptType, + request_id: str, + sampling_params_list: Sequence[OmniSamplingParams] | None = None, + output_modalities: list[str] | None = None, + ) -> AsyncGenerator[OmniRequestOutput, None]: + """Generate using inline diffusion engine (no stage worker subprocess). + + Eliminates Hop3 IPC overhead by running OmniDiffusion directly in the + orchestrator process. The blocking generate() call is offloaded to a + thread executor so the asyncio event loop remains responsive. + """ + _wall_start_ts = time.time() + + if sampling_params_list is None: + sampling_params_list = self.default_sampling_params_list + sp0 = sampling_params_list[0] + + stage = self.stage_list[0] + final_stage_id_for_e2e = 0 + + metrics = OrchestratorAggregator( + num_stages=1, + log_stats=self.log_stats, + wall_start_ts=_wall_start_ts, + final_stage_id_for_e2e=final_stage_id_for_e2e, + ) + metrics.stage_first_ts[0] = time.time() + + logger.info( + "[%s] Inline diffusion generate for request %s", + self._name, + request_id, + ) + + try: + loop = asyncio.get_running_loop() + results = await loop.run_in_executor( + None, + self._inline_engine.generate, + prompt, + sp0, + [request_id], + ) + + for result in results: + images = getattr(result, "images", None) or [] + finished = getattr(result, "finished", True) + + output_to_yield = OmniRequestOutput( + stage_id=0, + final_output_type=stage.final_output_type, + request_output=result, + images=images, + finished=finished, + ) + + metrics.stage_last_ts[0] = time.time() + yield output_to_yield + + try: + metrics.on_finalize_request( + final_stage_id_for_e2e, + request_id, + _wall_start_ts, + ) + metrics.build_and_log_summary() + except Exception as e: + logger.exception( + "[%s] Failed to finalize inline metrics: %s", + self._name, + e, + ) + + except (asyncio.CancelledError, GeneratorExit): + logger.info( + "[%s] Inline request %s cancelled.", + self._name, + request_id, + ) + raise + except Exception as e: + logger.exception( + "[%s] Inline diffusion failed for request %s: %s", + self._name, + request_id, + e, + ) + raise + async def _process_async_results( self, request_id: str, @@ -569,6 +673,7 @@ class AsyncOmni(OmniBase): raise RuntimeError(result) engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm") + if isinstance(engine_outputs, list): engine_outputs = engine_outputs[0] @@ -690,7 +795,8 @@ class AsyncOmni(OmniBase): @property def is_running(self) -> bool: - # Is None before the loop is started. + if self._inline_diffusion: + return self._inline_engine is not None return len(self._stage_in_queues) > 0 @property @@ -714,6 +820,10 @@ class AsyncOmni(OmniBase): return EngineDeadError() async def abort(self, request_id: str | Iterable[str]) -> None: + if self._inline_diffusion: + if self._inline_engine is not None: + self._inline_engine.engine.abort(request_id) + return None abort_task = {"type": OmniStageTaskType.ABORT, "request_id": request_id} for stage in self.stage_list: stage.submit(abort_task) diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index 598205971..c414b972b 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -40,6 +40,7 @@ from vllm_omni.entrypoints.omni_stage import OmniStage from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK, OmniStageTaskType from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load from vllm_omni.entrypoints.utils import ( + filter_dataclass_kwargs, get_final_stage_id_for_e2e, inject_omni_kv_config, load_and_resolve_stage_configs, @@ -314,6 +315,9 @@ class OmniBase: def _initialize_stages(self, model: str, kwargs: dict[str, Any]) -> None: """Initialize stage list management.""" + self._inline_diffusion = False + self._inline_engine = None + stage_init_timeout = kwargs.get("stage_init_timeout", 20) shm_threshold_bytes = kwargs.get("shm_threshold_bytes", 65536) init_timeout = kwargs.get("init_timeout", 300) @@ -361,6 +365,13 @@ class OmniBase: self.output_modalities = [st.final_output_type for st in self.stage_list] logger.info(f"[{self._name}] Loaded {len(self.stage_list)} stages") + # Phase 1 optimization: for a single diffusion stage in async mode, + # run the engine directly in the orchestrator process to eliminate + # the stage worker subprocess and its IPC serialization overhead. + if len(self.stage_list) == 1 and self.stage_list[0].stage_type == "diffusion" and self.is_async: + self._init_inline_diffusion_engine(model, self.stage_configs[0], kwargs) + return + if self.worker_backend == "ray": self._queue_cls = get_ray_queue_class() else: @@ -375,6 +386,69 @@ class OmniBase: # Set up RPC result checkers so that collective_rpc works self._setup_rpc_result_checkers() + def _init_inline_diffusion_engine( + self, + model: str, + stage_config: Any, + kwargs: dict[str, Any], + ) -> None: + """Initialize diffusion engine directly in the orchestrator process. + + For single-stage diffusion pipelines, this eliminates the stage worker + subprocess and the associated Hop3 IPC serialization overhead. + GPU workers for tensor parallelism are still spawned by the + DiffusionExecutor as separate processes. + """ + from vllm_omni.diffusion.data import OmniDiffusionConfig + from vllm_omni.entrypoints.omni_diffusion import OmniDiffusion + from vllm_omni.entrypoints.stage_utils import ( + _to_dict, + load_func_from_config, + set_stage_devices, + ) + + stage_id = stage_config.stage_id + engine_args = _to_dict(stage_config.engine_args) + runtime_cfg = _to_dict(getattr(stage_config, "runtime", {})) + + if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn": + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + try: + from vllm_omni.platforms import current_omni_platform + + device_type = current_omni_platform.device_type + set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type) + except Exception as e: + logger.warning("Device setup for inline diffusion failed: %s", e) + + engine_args = filter_dataclass_kwargs(OmniDiffusionConfig, engine_args) + engine_args.pop("model_stage", None) + engine_args.pop("model", None) + + cfg_kv_collect_func = load_func_from_config(getattr(stage_config, "cfg_kv_collect_func", None)) + + self._inline_engine = OmniDiffusion( + model=model, + stage_id=stage_id, + engine_input_source=getattr(stage_config, "engine_input_source", []), + cfg_kv_collect_func=cfg_kv_collect_func, + **engine_args, + ) + self._inline_diffusion = True + + # These attributes are normally set by AsyncOmni._wait_for_stages_ready + # but we skip that for inline mode. Set them to None since there is no + # LLM stage to provide them. + self.input_processor = None + self.io_processor = None + self.model_config = None + + logger.info( + "[%s] Inline diffusion mode active – stage worker subprocess bypassed", + self._name, + ) + def _is_async_chunk_enable(self, stage_args: list) -> bool: """get async chunk flag""" engine_args = getattr(stage_args[0], "engine_args", None) diff --git a/vllm_omni/entrypoints/omni_diffusion.py b/vllm_omni/entrypoints/omni_diffusion.py index 9937726e5..59ddf2eeb 100644 --- a/vllm_omni/entrypoints/omni_diffusion.py +++ b/vllm_omni/entrypoints/omni_diffusion.py @@ -1,9 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time import uuid from collections.abc import Sequence +from vllm.logger import init_logger from vllm.transformers_utils.config import get_hf_file_to_dict from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig @@ -12,6 +14,8 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType from vllm_omni.outputs import OmniRequestOutput +logger = init_logger(__name__) + class OmniDiffusion: """ @@ -108,6 +112,7 @@ class OmniDiffusion: sampling_params: OmniDiffusionSamplingParams, request_ids: list[str] = [], ) -> list[OmniRequestOutput]: + _t0 = time.perf_counter() if isinstance(prompts, (str, dict)): prompts = [prompts] else: @@ -118,7 +123,10 @@ class OmniDiffusion: request_ids.extend(f"{i + len(request_ids)}_{uuid.uuid4()}" for i in range(len(prompts) - len(request_ids))) request = OmniDiffusionRequest(prompts, sampling_params, request_ids) - return self._run_engine(request) + result = self._run_engine(request) + _t_ms = (time.perf_counter() - _t0) * 1000 + logger.info("OmniDiffusion.generate total: %.2f ms", _t_ms) + return result def _run_engine(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: return self.engine.step(request) diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 52e54f22f..3fa722b03 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -1119,7 +1119,6 @@ def _stage_worker( gen_outputs.extend(results) _gen_t1 = _time.time() _gen_ms = (_gen_t1 - _gen_t0) * 1000.0 - logger.debug(f"Generate done: batch={len(batch_tasks)}, req_ids={batch_request_ids}, gen_ms={_gen_ms:.1f}") # Group outputs per request id with fallback req_to_outputs: dict[Any, list[Any]] = {rid: [] for rid in batch_request_ids} @@ -1159,6 +1158,10 @@ def _stage_worker( _metrics.stage_stats = None try: use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes) + except Exception: + use_shm, payload = False, r_outputs + + try: if use_shm: out_q.put( { @@ -1186,10 +1189,6 @@ def _stage_worker( "metrics": _metrics, } ) - logger.debug( - "Enqueued result for request %s to downstream", - rid, - ) except Exception as e: logger.exception("Failed on batch %s: %s", batch_request_ids, e) _tb = traceback.format_exc() diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index f73df5ea4..b663eb52f 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -141,6 +141,7 @@ class OmniOpenAIServingVideo: ) result = await self._run_generation(prompt, gen_params, request_id, raw_request) + _t_encode_start = time.perf_counter() videos = self._extract_video_outputs(result) audios = self._extract_audio_outputs(result, expected_count=len(videos)) output_fps = fps or 24 @@ -161,6 +162,8 @@ class OmniOpenAIServingVideo: ) for idx, video in enumerate(videos) ] + _t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000 + logger.info("Video response encoding (MP4+base64): %.2f ms", _t_encode_ms) return VideoGenerationResponse(created=int(time.time()), data=video_data) def _resolve_model_name(self, raw_request: Request | None) -> str | None: