diff --git a/docs/.nav.yml b/docs/.nav.yml index fa06b53ec..a87c4c4f8 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -40,7 +40,7 @@ nav: - Features: - Sleep Mode: features/sleep_mode.md - Diffusion Features: - - Overview: user_guide/diffusion_acceleration.md + - Acceleration Overview: user_guide/diffusion_acceleration.md - TeaCache: user_guide/diffusion/teacache.md - Cache-DiT: user_guide/diffusion/cache_dit_acceleration.md - Quantization: @@ -48,6 +48,7 @@ nav: - FP8: user_guide/diffusion/quantization/fp8.md - Parallelism Acceleration: user_guide/diffusion/parallelism_acceleration.md - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md + - LoRA: user_guide/diffusion/lora.md - ComfyUI: features/comfyui.md - Developer Guide: - General: diff --git a/docs/user_guide/diffusion/lora.md b/docs/user_guide/diffusion/lora.md new file mode 100644 index 000000000..e45c033b8 --- /dev/null +++ b/docs/user_guide/diffusion/lora.md @@ -0,0 +1,63 @@ +# LoRA (Low-Rank Adaptation) Guide + +LoRA (Low-Rank Adaptation) enables fine-tuning diffusion models by adding trainable low-rank matrices to existing model weights. vLLM-Omni currently supports PEFT-style LoRA adapters, allowing you to customize model behavior without modifying the base model weights. + +## Overview + +LoRA adapters are lightweight, model-specific fine-tuning weights that can be dynamically loaded and applied to diffusion models. vLLM-Omni uses a unified LoRA handling mechanism similar to vLLM with LRU cache management. + +## LoRA Adapter Format + +LoRA adapters must be in **PEFT (Parameter-Efficient Fine-Tuning)** format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + +The `adapter_config.json` file contains metadata about the LoRA adapter, including: +- `r`: LoRA rank +- `lora_alpha`: LoRA alpha scaling factor +- `target_modules`: List of module names to apply LoRA to + +## Quick Start + +### Offline Inference + +#### Pre-loaded LoRA + +Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests: + +```python +from vllm_omni import Omni +from vllm_omni.lora.request import LoRARequest + +lora_path="/path/to/lora_adapter" + +omni = Omni( + model="stabilityai/stable-diffusion-3.5-medium", + lora_path=lora_path +) + +lora_request = LoRARequest( + lora_name="preloaded", + lora_int_id=1, + lora_path=lora_path +) + +outputs = omni.generate( + prompt="A piece of cheesecake", + lora_request=lora_request, + lora_scale=2.0, # optional arg, default 1.0 +) +``` + +!!! note "Server-side Path Requirement" + The LoRA adapter path (`local_path`) must be readable on the **server** machine. If your client and server are on different machines, ensure the LoRA adapter is accessible via a shared mount or copied to the server. + + +## See Also + +- [Text-to-Image Offline Example](../examples/offline_inference/text_to_image.md#lora) - Complete offline LoRA example +- [Text-to-Image Online Example](../examples/online_serving/text_to_image.md#lora) - Complete online LoRA example diff --git a/docs/user_guide/examples/offline_inference/lora_inference.md b/docs/user_guide/examples/offline_inference/lora_inference.md deleted file mode 100644 index f8a961dc9..000000000 --- a/docs/user_guide/examples/offline_inference/lora_inference.md +++ /dev/null @@ -1,108 +0,0 @@ -# LoRA Inference(Diffusion) - -Source . - - -This directory contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. -The example uses the `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni. - -## Overview - -Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism: - -- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache) -- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request - -Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated. - -## Usage - -### Pre-loaded LoRA (via --lora-path) - -Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --lora-path /path/to/lora/ \ - --lora-scale 1.0 \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_preloaded.png -``` - -**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request. - -### Per-request LoRA (via --lora-request-path) - -Load a LoRA adapter on-demand for each request: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --lora-request-path /path/to/lora/ \ - --lora-scale 1.0 \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_per_request.png -``` - -### No LoRA - -If no LoRA request is provided, we will use the base model without any LoRA adapters: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_no_lora.png -``` - -## Parameters - -### LoRA Parameters - -- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path) -- `--lora-request-path`: Path to LoRA adapter folder for per-request loading -- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path. -- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. - -### Standard Parameters - -- `--prompt`: Text prompt for image generation (required) -- `--seed`: Random seed for reproducibility (default: 42) -- `--height`: Image height in pixels (default: 1024) -- `--width`: Image width in pixels (default: 1024) -- `--num-inference-steps`: Number of denoising steps (default: 50) -- `--output`: Output file path (default: `lora_output.png`) - -## How LoRA Works - -All LoRA adapters are handled uniformly: - -1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path -2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request -3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated - -The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned). - -## LoRA Adapter Format - -LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: - -``` -lora_adapter/ -├── adapter_config.json -└── adapter_model.safetensors -``` - -## Example materials - -??? abstract "lora_inference.py" - ``````py - --8<-- "examples/offline_inference/lora_inference/lora_inference.py" - `````` diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index 5451fe223..a45108cf4 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -108,6 +108,36 @@ Key arguments: > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. +## LoRA + +This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. Pass `--lora-path` to use a LoRA adapter and optionally `--lora-scale` (default 1.0); omit it to use the base model only. + +### Basic usage with LoRA + +```bash +python text_to_image.py \ + --model Tongyi-MAI/Z-Image-Turbo \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora/ \ + --lora-scale 1.0 \ + --output output.png +``` + +### LoRA parameters + +- `--lora-path`: Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation. +- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. + +### LoRA adapter format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + ## Web UI Demo Launch the gradio demo: diff --git a/docs/user_guide/examples/online_serving/lora_inference.md b/docs/user_guide/examples/online_serving/lora_inference.md deleted file mode 100644 index f399d41aa..000000000 --- a/docs/user_guide/examples/online_serving/lora_inference.md +++ /dev/null @@ -1,72 +0,0 @@ -# LoRA Inference(Diffusion) - -Source . - - -This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API. - -> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). -> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here. - -## Start Server - -```bash -# Pick a diffusion model (examples) -# export MODEL=stabilityai/stable-diffusion-3.5-medium -# export MODEL=Qwen/Qwen-Image - -bash run_server.sh -``` - -## Call API (curl) - -```bash -# Required: local LoRA folder on the server -export LORA_PATH=/path/to/lora_adapter - -# Optional -export SERVER=http://localhost:8091 -export PROMPT="A piece of cheesecake" -export LORA_NAME=my_lora -export LORA_SCALE=1.0 -# Optional: if omitted, the server derives a stable id from LORA_PATH. -# export LORA_INT_ID=123 - -bash run_curl_lora_inference.sh -``` - -## Call API (Python) - -```bash -python openai_chat_client.py \ - --prompt "A piece of cheesecake" \ - --lora-path /path/to/lora_adapter \ - --lora-name my_lora \ - --lora-scale 1.0 \ - --output output.png -``` - -## LoRA Format - -LoRA adapters should be in PEFT format, for example: - -``` -lora_adapter/ -├── adapter_config.json -└── adapter_model.safetensors -``` - -## Example materials - -??? abstract "openai_chat_client.py" - ``````py - --8<-- "examples/online_serving/lora_inference/openai_chat_client.py" - `````` -??? abstract "run_curl_lora_inference.sh" - ``````sh - --8<-- "examples/online_serving/lora_inference/run_curl_lora_inference.sh" - `````` -??? abstract "run_server.sh" - ``````sh - --8<-- "examples/online_serving/lora_inference/run_server.sh" - `````` diff --git a/docs/user_guide/examples/online_serving/text_to_image.md b/docs/user_guide/examples/online_serving/text_to_image.md index 2de2b4fcb..be8c7d894 100644 --- a/docs/user_guide/examples/online_serving/text_to_image.md +++ b/docs/user_guide/examples/online_serving/text_to_image.md @@ -61,6 +61,59 @@ python gradio_demo.py # Visit http://localhost:7860 ``` +## LoRA + +This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). + +### Using Python Client with LoRA + +```bash +python openai_chat_client.py \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora_adapter \ + --lora-name my_lora \ + --lora-scale 1.0 \ + --output output.png +``` + +### Using curl with LoRA (Images API) + +The `/v1/images/generations` endpoint supports a `lora` field in the request body: + +```bash +curl -X POST http://localhost:8091/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A piece of cheesecake", + "size": "1024x1024", + "seed": 42, + "lora": { + "name": "my_lora", + "local_path": "/path/to/lora_adapter", + "scale": 1.0 + } + }' | jq -r '.data[0].b64_json' | base64 -d > output.png +``` + +### LoRA Parameters + +| Parameter | Type | Description | +| ------------- | ------ | ---------------------------------------------------------------- | +| `name` | str | LoRA adapter name (optional, defaults to path stem) | +| `local_path` | str | Server-local path to LoRA adapter folder (PEFT format, required) | +| `scale` | float | LoRA scale factor (default: 1.0) | +| `int_id` | int | LoRA integer ID for caching (optional, derived from path if not provided) | + +### LoRA Adapter Format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + ## Request Format ### Simple Text Generation diff --git a/examples/offline_inference/lora_inference/README.md b/examples/offline_inference/lora_inference/README.md deleted file mode 100644 index d53b45196..000000000 --- a/examples/offline_inference/lora_inference/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# LoRA Inference(Diffusion) - -This directory contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. -The example uses the `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni. - -## Overview - -Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism: - -- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache) -- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request - -Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated. - -## Usage - -### Pre-loaded LoRA (via --lora-path) - -Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --lora-path /path/to/lora/ \ - --lora-scale 1.0 \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_preloaded.png -``` - -**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request. - -### Per-request LoRA (via --lora-request-path) - -Load a LoRA adapter on-demand for each request: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --lora-request-path /path/to/lora/ \ - --lora-scale 1.0 \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_per_request.png -``` - -### No LoRA - -If no LoRA request is provided, we will use the base model without any LoRA adapters: - -```bash -python -m examples.offline_inference.lora_inference.lora_inference \ - --prompt "A piece of cheesecake" \ - --num-inference-steps 50 \ - --height 1024 \ - --width 1024 \ - --output output_no_lora.png -``` - -## Parameters - -### LoRA Parameters - -- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path) -- `--lora-request-path`: Path to LoRA adapter folder for per-request loading -- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path. -- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. - -### Standard Parameters - -- `--prompt`: Text prompt for image generation (required) -- `--seed`: Random seed for reproducibility (default: 42) -- `--height`: Image height in pixels (default: 1024) -- `--width`: Image width in pixels (default: 1024) -- `--num-inference-steps`: Number of denoising steps (default: 50) -- `--output`: Output file path (default: `lora_output.png`) - -## How LoRA Works - -All LoRA adapters are handled uniformly: - -1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path -2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request -3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated - -The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned). - -## LoRA Adapter Format - -LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: - -``` -lora_adapter/ -├── adapter_config.json -└── adapter_model.safetensors -``` diff --git a/examples/offline_inference/lora_inference/lora_inference.py b/examples/offline_inference/lora_inference/lora_inference.py deleted file mode 100644 index a7803598b..000000000 --- a/examples/offline_inference/lora_inference/lora_inference.py +++ /dev/null @@ -1,147 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse -from pathlib import Path - -from vllm_omni.entrypoints.omni import Omni -from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.lora.request import LoRARequest -from vllm_omni.lora.utils import stable_lora_int_id - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Generate images with LoRA adapters.") - parser.add_argument("--model", default="stabilityai/stable-diffusion-3.5-medium", help="Model name or path.") - parser.add_argument("--prompt", required=True, help="Text prompt for image generation.") - parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic results.") - parser.add_argument("--height", type=int, default=1024, help="Height of generated image.") - parser.add_argument("--width", type=int, default=1024, help="Width of generated image.") - parser.add_argument( - "--num-inference-steps", - type=int, - default=50, - help="Number of denoising steps for the diffusion sampler.", - ) - parser.add_argument( - "--output", - type=str, - default="lora_output.png", - help="Path to save the generated image (PNG).", - ) - parser.add_argument( - "--lora-path", - type=str, - default=None, - help="Path to LoRA adapter folder to pre-load at initialization (PEFT format). " - "Note: pre-loading populates the cache; you still need to pass a lora_request to activate it.", - ) - parser.add_argument( - "--lora-request-path", - type=str, - default=None, - help="Path to LoRA adapter folder for per-request activation (dynamic LoRA). " - "If --lora-request-id is not provided, a stable ID will be derived from this path.", - ) - parser.add_argument( - "--lora-request-id", - type=int, - default=None, - help="Integer ID for the LoRA adapter (for dynamic LoRA). " - "If not provided and --lora-request-path is set, will derive a stable ID from the path.", - ) - parser.add_argument( - "--lora-scale", - type=float, - default=1.0, - help="Scale factor for LoRA weights (default: 1.0).", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - - model = args.model - - omni_kwargs = {} - - if args.lora_path: - omni_kwargs["lora_path"] = args.lora_path - print(f"Using static LoRA from: {args.lora_path}") - - omni = Omni(model=model, **omni_kwargs) - - lora_request = None - if args.lora_request_path: - if args.lora_request_id is None: - lora_request_id = stable_lora_int_id(args.lora_request_path) - else: - lora_request_id = args.lora_request_id - - lora_name = Path(args.lora_request_path).stem - lora_request = LoRARequest( - lora_name=lora_name, - lora_int_id=lora_request_id, - lora_path=args.lora_request_path, - ) - print(f"Using per-request LoRA: name={lora_name}, id={lora_request_id}, scale={args.lora_scale}") - elif args.lora_path: - # pre-loaded LoRA - lora_request_id = stable_lora_int_id(args.lora_path) - lora_request = LoRARequest( - lora_name="preloaded", - lora_int_id=lora_request_id, - lora_path=args.lora_path, - ) - print(f"Activating pre-loaded LoRA: id={lora_request_id}, scale={args.lora_scale}") - - sampling_params = OmniDiffusionSamplingParams( - height=args.height, - width=args.width, - num_inference_steps=args.num_inference_steps, - ) - - if lora_request: - sampling_params.lora_request = lora_request - sampling_params.lora_scale = args.lora_scale - - outputs = omni.generate(args.prompt, sampling_params) - - if not outputs or len(outputs) == 0: - raise ValueError("No output generated from omni.generate()") - - if isinstance(outputs, list): - first_output = outputs[0] - else: - first_output = outputs - - images = None - if hasattr(first_output, "images") and first_output.images: - images = first_output.images - elif hasattr(first_output, "request_output") and first_output.request_output: - req_out = first_output.request_output - if isinstance(req_out, list) and len(req_out) > 0: - req_out = req_out[0] - if hasattr(req_out, "images") and req_out.images: - images = req_out.images - - if not images: - raise ValueError("No images found in request_output") - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - suffix = output_path.suffix or ".png" - stem = output_path.stem or "lora_output" - if len(images) <= 1: - images[0].save(output_path) - print(f"Saved generated image to {output_path}") - else: - for idx, img in enumerate(images): - save_path = output_path.parent / f"{stem}_{idx}{suffix}" - img.save(save_path) - print(f"Saved generated image to {save_path}") - - -if __name__ == "__main__": - main() diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index 2bfae8472..ff2dda010 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -133,6 +133,36 @@ python text_to_image.py \ > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. +## LoRA + +This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. Pass `--lora-path` to use a LoRA adapter and optionally `--lora-scale` (default 1.0); omit it to use the base model only. + +### Basic usage with LoRA + +```bash +python text_to_image.py \ + --model Tongyi-MAI/Z-Image-Turbo \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora/ \ + --lora-scale 1.0 \ + --output output.png +``` + +### LoRA parameters + +- `--lora-path`: Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation. +- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. + +### LoRA adapter format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + ## Web UI Demo Launch the gradio demo: diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 04f66863c..5e1804780 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -12,6 +12,8 @@ import torch from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id from vllm_omni.outputs import OmniRequestOutput from vllm_omni.platforms import current_omni_platform @@ -160,6 +162,28 @@ def parse_args() -> argparse.Namespace: default=1, help="Number of GPUs used for tensor parallelism (TP) inside the DiT.", ) + parser.add_argument( + "--lora-path", + type=str, + default=None, + help="Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation.", + ) + parser.add_argument( + "--lora-scale", + type=float, + default=1.0, + help="Scale factor for LoRA weights (default: 1.0).", + ) + parser.add_argument( + "--vae_use_slicing", + action="store_true", + help="Enable VAE slicing for memory optimization.", + ) + parser.add_argument( + "--vae_use_tiling", + action="store_true", + help="Enable VAE tiling for memory optimization.", + ) parser.add_argument( "--vae-patch-parallel-size", type=int, @@ -241,6 +265,12 @@ def main(): # Check if profiling is requested via environment variable profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) + # Prepare LoRA kwargs for Omni initialization + lora_args: dict[str, Any] = {} + if args.lora_path: + lora_args["lora_path"] = args.lora_path + print(f"Using LoRA from: {args.lora_path}") + # Build quantization kwargs: use quantization_config dict when # ignored_layers is specified so the list flows through OmniDiffusionConfig quant_kwargs: dict[str, Any] = {} @@ -253,18 +283,19 @@ def main(): elif args.quantization: quant_kwargs["quantization"] = args.quantization - # Initialize Omni with model-specific settings omni_kwargs = { "model": args.model, "enable_layerwise_offload": args.enable_layerwise_offload, + "layerwise_num_gpu_layers": args.layerwise_num_gpu_layers, "vae_use_slicing": args.vae_use_slicing, "vae_use_tiling": args.vae_use_tiling, - "cache_backend": cache_backend, + "cache_backend": args.cache_backend, "cache_config": cache_config, "enable_cache_dit_summary": args.enable_cache_dit_summary, "parallel_config": parallel_config, "enforce_eager": args.enforce_eager, "enable_cpu_offload": args.enable_cpu_offload, + **lora_args, **quant_kwargs, } if use_nextstep: @@ -292,8 +323,20 @@ def main(): ) print(f" CPU offload: {args.enable_cpu_offload}") print(f" Image size: {args.width}x{args.height}") + if args.lora_path: + print(f" LoRA: scale={args.lora_scale}") print(f"{'=' * 60}\n") + # Build LoRA request when --lora-path is set + lora_request = None + if args.lora_path: + lora_request_id = stable_lora_int_id(args.lora_path) + lora_request = LoRARequest( + lora_name=Path(args.lora_path).stem, + lora_int_id=lora_request_id, + lora_path=args.lora_path, + ) + generation_start = time.perf_counter() extra_args = { @@ -302,6 +345,10 @@ def main(): "use_norm": args.use_norm, } + if lora_request: + extra_args["lora_request"] = lora_request + extra_args["lora_scale"] = args.lora_scale + outputs = omni.generate( { "prompt": args.prompt, diff --git a/examples/online_serving/lora_inference/README.md b/examples/online_serving/lora_inference/README.md deleted file mode 100644 index 3017d9553..000000000 --- a/examples/online_serving/lora_inference/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# LoRA Inference(Diffusion) - -This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API. - -> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). -> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here. - -## Start Server - -```bash -# Pick a diffusion model (examples) -# export MODEL=stabilityai/stable-diffusion-3.5-medium -# export MODEL=Qwen/Qwen-Image - -bash run_server.sh -``` - -## Call API (curl) - -```bash -# Required: local LoRA folder on the server -export LORA_PATH=/path/to/lora_adapter - -# Optional -export SERVER=http://localhost:8091 -export PROMPT="A piece of cheesecake" -export LORA_NAME=my_lora -export LORA_SCALE=1.0 -# Optional: if omitted, the server derives a stable id from LORA_PATH. -# export LORA_INT_ID=123 - -bash run_curl_lora_inference.sh -``` - -## Call API (Python) - -```bash -python openai_chat_client.py \ - --prompt "A piece of cheesecake" \ - --lora-path /path/to/lora_adapter \ - --lora-name my_lora \ - --lora-scale 1.0 \ - --output output.png -``` - -## LoRA Format - -LoRA adapters should be in PEFT format, for example: - -``` -lora_adapter/ -├── adapter_config.json -└── adapter_model.safetensors -``` diff --git a/examples/online_serving/lora_inference/openai_chat_client.py b/examples/online_serving/lora_inference/openai_chat_client.py deleted file mode 100644 index e24d2fdf6..000000000 --- a/examples/online_serving/lora_inference/openai_chat_client.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -""" -OpenAI-compatible chat client for diffusion LoRA inference. - -Example: - python openai_chat_client.py \ - --server http://localhost:8091 \ - --prompt "A piece of cheesecake" \ - --lora-path /path/to/lora_adapter \ - --lora-name my_lora \ - --lora-scale 1.0 \ - --output output.png -""" - -import argparse -import base64 -from pathlib import Path - -import requests - - -def generate_image( - *, - prompt: str, - server_url: str, - height: int | None, - width: int | None, - num_inference_steps: int | None, - seed: int | None, - lora_name: str | None, - lora_path: str | None, - lora_scale: float | None, - lora_int_id: int | None, -) -> bytes | None: - messages = [{"role": "user", "content": prompt}] - - extra_body: dict = {} - if height is not None: - extra_body["height"] = height - if width is not None: - extra_body["width"] = width - if num_inference_steps is not None: - extra_body["num_inference_steps"] = num_inference_steps - if seed is not None: - extra_body["seed"] = seed - - if lora_path: - lora_body: dict = { - "local_path": lora_path, - "name": lora_name or Path(lora_path).stem, - } - if lora_scale is not None: - lora_body["scale"] = float(lora_scale) - if lora_int_id is not None: - lora_body["int_id"] = int(lora_int_id) - extra_body["lora"] = lora_body - - payload = {"messages": messages} - if extra_body: - payload["extra_body"] = extra_body - - response = requests.post( - f"{server_url}/v1/chat/completions", - headers={"Content-Type": "application/json"}, - json=payload, - timeout=300, - ) - response.raise_for_status() - data = response.json() - - content = data["choices"][0]["message"]["content"] - if isinstance(content, list) and content: - image_url = content[0].get("image_url", {}).get("url", "") - if image_url.startswith("data:image"): - _, b64_data = image_url.split(",", 1) - return base64.b64decode(b64_data) - - raise RuntimeError(f"Unexpected response format: {content!r}") - - -def main() -> None: - parser = argparse.ArgumentParser(description="Diffusion LoRA OpenAI chat client") - parser.add_argument("--server", default="http://localhost:8091", help="Server URL") - parser.add_argument("--prompt", default="A piece of cheesecake", help="Text prompt") - parser.add_argument("--output", default="lora_online_output.png", help="Output image path") - - parser.add_argument("--height", type=int, default=1024, help="Image height") - parser.add_argument("--width", type=int, default=1024, help="Image width") - parser.add_argument("--steps", type=int, default=50, help="num_inference_steps") - parser.add_argument("--seed", type=int, default=42, help="Random seed") - - parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)") - parser.add_argument("--lora-name", default=None, help="LoRA name (optional)") - parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale") - parser.add_argument( - "--lora-int-id", - type=int, - default=None, - help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.", - ) - - args = parser.parse_args() - - image_bytes = generate_image( - prompt=args.prompt, - server_url=args.server, - height=args.height, - width=args.width, - num_inference_steps=args.steps, - seed=args.seed, - lora_name=args.lora_name, - lora_path=args.lora_path, - lora_scale=args.lora_scale if args.lora_path else None, - lora_int_id=args.lora_int_id if args.lora_path else None, - ) - - out_path = Path(args.output) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_bytes(image_bytes) - print(f"Saved: {out_path} ({len(image_bytes) / 1024:.1f} KiB)") - - -if __name__ == "__main__": - main() diff --git a/examples/online_serving/lora_inference/run_curl_lora_inference.sh b/examples/online_serving/lora_inference/run_curl_lora_inference.sh deleted file mode 100755 index 14a074fbf..000000000 --- a/examples/online_serving/lora_inference/run_curl_lora_inference.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Online diffusion LoRA inference via OpenAI-compatible chat API. - -SERVER="${SERVER:-http://localhost:8091}" -PROMPT="${PROMPT:-A piece of cheesecake}" - -LORA_PATH="${LORA_PATH:-}" -LORA_NAME="${LORA_NAME:-lora}" -LORA_SCALE="${LORA_SCALE:-1.0}" -LORA_INT_ID="${LORA_INT_ID:-}" - -HEIGHT="${HEIGHT:-1024}" -WIDTH="${WIDTH:-1024}" -NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-50}" -SEED="${SEED:-42}" - -CURRENT_TIME=$(date +%Y%m%d%H%M%S) -OUTPUT="${OUTPUT:-lora_online_output_${CURRENT_TIME}.png}" - -if [ -z "$LORA_PATH" ]; then - echo "ERROR: LORA_PATH is required (must be a server-local path)." - exit 1 -fi - -echo "Generating image with LoRA..." -echo "Server: $SERVER" -echo "Prompt: $PROMPT" -echo "LoRA: name=$LORA_NAME id=${LORA_INT_ID:-auto} scale=$LORA_SCALE path=$LORA_PATH" -echo "Output: $OUTPUT" - -LORA_INT_ID_FIELD="" -if [ -n "$LORA_INT_ID" ]; then - LORA_INT_ID_FIELD=", \"int_id\": $LORA_INT_ID" -fi - -curl -s "$SERVER/v1/chat/completions" \ - -H "Content-Type: application/json" \ - -d "{ - \"messages\": [ - {\"role\": \"user\", \"content\": \"$PROMPT\"} - ], - \"extra_body\": { - \"height\": $HEIGHT, - \"width\": $WIDTH, - \"num_inference_steps\": $NUM_INFERENCE_STEPS, - \"seed\": $SEED, - \"lora\": { - \"name\": \"$LORA_NAME\", - \"local_path\": \"$LORA_PATH\", - \"scale\": $LORA_SCALE$LORA_INT_ID_FIELD - } - } - }" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT" - -if [ -f "$OUTPUT" ]; then - echo "Image saved to: $OUTPUT" - echo "Size: $(du -h "$OUTPUT" | cut -f1)" -else - echo "Failed to generate image" - exit 1 -fi diff --git a/examples/online_serving/lora_inference/run_server.sh b/examples/online_serving/lora_inference/run_server.sh deleted file mode 100755 index 3233dd773..000000000 --- a/examples/online_serving/lora_inference/run_server.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Online diffusion serving with vLLM-Omni (OpenAI-compatible API). - -MODEL="${MODEL:-stabilityai/stable-diffusion-3.5-medium}" -PORT="${PORT:-8091}" - -echo "Starting vLLM-Omni diffusion server..." -echo "Model: $MODEL" -echo "Port: $PORT" - -if [ -z "${VLLM_BIN:-}" ]; then - if command -v vllm-omni >/dev/null 2>&1; then - VLLM_BIN="vllm-omni" - else - VLLM_BIN="vllm" - fi -fi - -"$VLLM_BIN" serve "$MODEL" --omni \ - --port "$PORT" diff --git a/examples/online_serving/text_to_image/README.md b/examples/online_serving/text_to_image/README.md index 744b7b292..140036d00 100644 --- a/examples/online_serving/text_to_image/README.md +++ b/examples/online_serving/text_to_image/README.md @@ -58,6 +58,59 @@ python gradio_demo.py # Visit http://localhost:7860 ``` +## LoRA + +This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). + +### Using Python Client with LoRA + +```bash +python openai_chat_client.py \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora_adapter \ + --lora-name my_lora \ + --lora-scale 1.0 \ + --output output.png +``` + +### Using curl with LoRA (Images API) + +The `/v1/images/generations` endpoint supports a `lora` field in the request body: + +```bash +curl -X POST http://localhost:8091/v1/images/generations \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "A piece of cheesecake", + "size": "1024x1024", + "seed": 42, + "lora": { + "name": "my_lora", + "local_path": "/path/to/lora_adapter", + "scale": 1.0 + } + }' | jq -r '.data[0].b64_json' | base64 -d > output.png +``` + +### LoRA Parameters + +| Parameter | Type | Description | +| ------------- | ------ | ---------------------------------------------------------------- | +| `name` | str | LoRA adapter name (optional, defaults to path stem) | +| `local_path` | str | Server-local path to LoRA adapter folder (PEFT format, required) | +| `scale` | float | LoRA scale factor (default: 1.0) | +| `int_id` | int | LoRA integer ID for caching (optional, derived from path if not provided) | + +### LoRA Adapter Format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + ## Request Format ### Simple Text Generation diff --git a/examples/online_serving/text_to_image/openai_chat_client.py b/examples/online_serving/text_to_image/openai_chat_client.py index 7beac1acc..828827aba 100644 --- a/examples/online_serving/text_to_image/openai_chat_client.py +++ b/examples/online_serving/text_to_image/openai_chat_client.py @@ -24,6 +24,10 @@ def generate_image( seed: int | None = None, negative_prompt: str | None = None, num_outputs_per_prompt: int = 1, + lora_path: str | None = None, + lora_name: str | None = None, + lora_scale: float | None = None, + lora_int_id: int | None = None, ) -> bytes | None: """Generate an image using the images generation API. @@ -37,6 +41,10 @@ def generate_image( seed: Random seed negative_prompt: Negative prompt num_outputs_per_prompt: Number of images to generate + lora_path: Server-local LoRA adapter folder path (PEFT format) + lora_name: LoRA name (optional, defaults to path stem) + lora_scale: LoRA scale factor (default: 1.0) + lora_int_id: LoRA integer ID (optional, derived from path if not provided) Returns: Image bytes or None if failed @@ -63,6 +71,18 @@ def generate_image( if seed is not None: payload["seed"] = seed + # Add LoRA if provided + if lora_path: + lora_body: dict = { + "local_path": lora_path, + "name": lora_name or Path(lora_path).stem, + } + if lora_scale is not None: + lora_body["scale"] = float(lora_scale) + if lora_int_id is not None: + lora_body["int_id"] = int(lora_int_id) + payload["lora"] = lora_body + try: response = requests.post( f"{server_url}/v1/images/generations", @@ -99,6 +119,16 @@ def main(): parser.add_argument("--seed", type=int, default=0, help="Random seed") parser.add_argument("--negative", help="Negative prompt") + parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)") + parser.add_argument("--lora-name", default=None, help="LoRA name (optional)") + parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale") + parser.add_argument( + "--lora-int-id", + type=int, + default=None, + help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.", + ) + args = parser.parse_args() print(f"Generating image for: {args.prompt}") @@ -112,6 +142,10 @@ def main(): true_cfg_scale=args.cfg_scale, seed=args.seed, negative_prompt=args.negative, + lora_path=args.lora_path, + lora_name=args.lora_name, + lora_scale=args.lora_scale if args.lora_path else None, + lora_int_id=args.lora_int_id if args.lora_path else None, ) if image_bytes: