diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index c31d09825..96ea35a4b 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -347,6 +347,9 @@ def main(): ) print("Pipeline loaded") + # Check if profiling is requested via environment variable + profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) + # Time profiling for generation print(f"\n{'=' * 60}") print("Generation Configuration:") @@ -379,6 +382,10 @@ def main(): "resolution": args.resolution, } + if profiler_enabled: + print("[Profiler] Starting profiling...") + omni.start_profile() + outputs = omni.generate(**generate_kwargs) generation_end = time.perf_counter() generation_time = generation_end - generation_start @@ -386,6 +393,23 @@ def main(): # Print profiling results print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)") + if profiler_enabled: + print("\n[Profiler] Stopping profiler and collecting results...") + profile_results = omni.stop_profile() + if profile_results and isinstance(profile_results, dict): + traces = profile_results.get("traces", []) + print("\n" + "=" * 60) + print("PROFILING RESULTS:") + for rank, trace in enumerate(traces): + print(f"\nRank {rank}:") + if trace: + print(f" • Trace: {trace}") + if not traces: + print(" No traces collected.") + print("=" * 60) + else: + print("[Profiler] No valid profiling data returned.") + if not outputs: raise ValueError("No output generated from omni.generate()") logger.info("Outputs: %s", outputs) diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 023255989..f40eebaef 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -19,6 +19,7 @@ Usage: """ import argparse +import os from pathlib import Path import numpy as np @@ -95,6 +96,9 @@ def main(): vae_use_slicing = is_npu() vae_use_tiling = is_npu() + # Check if profiling is requested via environment variable + profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) + omni = Omni( model=args.model, vae_use_slicing=vae_use_slicing, @@ -103,6 +107,10 @@ def main(): flow_shift=args.flow_shift, ) + if profiler_enabled: + print("[Profiler] Starting profiling...") + omni.start_profile() + # omni.generate() returns Generator[OmniRequestOutput, None, None] frames = omni.generate( args.prompt, @@ -178,6 +186,23 @@ def main(): export_to_video(video_array, str(output_path), fps=args.fps) print(f"Saved generated video to {output_path}") + if profiler_enabled: + print("\n[Profiler] Stopping profiler and collecting results...") + profile_results = omni.stop_profile() + if profile_results and isinstance(profile_results, dict): + traces = profile_results.get("traces", []) + print("\n" + "=" * 60) + print("PROFILING RESULTS:") + for rank, trace in enumerate(traces): + print(f"\nRank {rank}:") + if trace: + print(f" • Trace: {trace}") + if not traces: + print(" No traces collected.") + print("=" * 60) + else: + print("[Profiler] No valid profiling data returned.") + if __name__ == "__main__": main() diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 9f06a7105..fdb7c8a0a 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import os import time from pathlib import Path @@ -148,6 +149,9 @@ def main(): tensor_parallel_size=args.tensor_parallel_size, ) + # Check if profiling is requested via environment variable + profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) + omni = Omni( model=args.model, vae_use_slicing=vae_use_slicing, @@ -158,6 +162,10 @@ def main(): enforce_eager=args.enforce_eager, ) + if profiler_enabled: + print("[Profiler] Starting profiling...") + omni.start_profile() + # Time profiling for generation print(f"\n{'=' * 60}") print("Generation Configuration:") @@ -189,6 +197,23 @@ def main(): # Print profiling results print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)") + if profiler_enabled: + print("\n[Profiler] Stopping profiler and collecting results...") + profile_results = omni.stop_profile() + if profile_results and isinstance(profile_results, dict): + traces = profile_results.get("traces", []) + print("\n" + "=" * 60) + print("PROFILING RESULTS:") + for rank, trace in enumerate(traces): + print(f"\nRank {rank}:") + if trace: + print(f" • Trace: {trace}") + if not traces: + print(" No traces collected.") + print("=" * 60) + else: + print("[Profiler] No valid profiling data returned.") + # Extract images from OmniRequestOutput # omni.generate() returns list[OmniRequestOutput], extract images from the first output if not outputs or len(outputs) == 0: diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index e11d60eb7..f50692ce6 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import os from pathlib import Path import numpy as np @@ -46,6 +47,9 @@ def main(): vae_use_slicing = is_npu() vae_use_tiling = is_npu() + # Check if profiling is requested via environment variable + profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) + omni = Omni( model=args.model, vae_use_slicing=vae_use_slicing, @@ -54,6 +58,10 @@ def main(): flow_shift=args.flow_shift, ) + if profiler_enabled: + print("[Profiler] Starting profiling...") + omni.start_profile() + frames = omni.generate( args.prompt, negative_prompt=args.negative_prompt, @@ -126,6 +134,23 @@ def main(): export_to_video(video_array, str(output_path), fps=args.fps) print(f"Saved generated video to {output_path}") + if profiler_enabled: + print("\n[Profiler] Stopping profiler and collecting results...") + profile_results = omni.stop_profile() + if profile_results and isinstance(profile_results, dict): + traces = profile_results.get("traces", []) + print("\n" + "=" * 60) + print("PROFILING RESULTS:") + for rank, trace in enumerate(traces): + print(f"\nRank {rank}:") + if trace: + print(f" • Trace: {trace}") + if not traces: + print(" No traces collected.") + print("=" * 60) + else: + print("[Profiler] No valid profiling data returned.") + if __name__ == "__main__": main() diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 1948b40a3..0f21d4b28 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp +import os import time import weakref from collections.abc import Callable, Iterable @@ -290,6 +291,125 @@ class DiffusionEngine: def add_req_and_wait_for_response(self, requests: list[OmniDiffusionRequest]): return scheduler.add_req(requests) + def start_profile(self, trace_filename: str | None = None) -> None: + """ + Start torch profiling on all diffusion workers. + + Creates a directory (if needed) and sets up a base filename template + for per-rank profiler traces (typically saved as