[Doc] format lora related docs for the user's end (#1009)

Signed-off-by: AndyZhou952 <jzhoubc@connect.ust.hk>
Signed-off-by: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com>
This commit is contained in:
Andy Zhou authored and GitHub committed 2026-02-26 16:48:29 +08:00
1 parent caed95fb45
commit a2d5b61703
16 files changed
+314 -687

No files matched your search

@@ -1,54 +0,0 @@
# LoRA Inference(Diffusion)
This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API.
> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here.
## Start Server
```bash
# Pick a diffusion model (examples)
# export MODEL=stabilityai/stable-diffusion-3.5-medium
# export MODEL=Qwen/Qwen-Image
bash run_server.sh
```
## Call API (curl)
```bash
# Required: local LoRA folder on the server
export LORA_PATH=/path/to/lora_adapter
# Optional
export SERVER=http://localhost:8091
export PROMPT="A piece of cheesecake"
export LORA_NAME=my_lora
export LORA_SCALE=1.0
# Optional: if omitted, the server derives a stable id from LORA_PATH.
# export LORA_INT_ID=123
bash run_curl_lora_inference.sh
```
## Call API (Python)
```bash
python openai_chat_client.py \
--prompt "A piece of cheesecake" \
--lora-path /path/to/lora_adapter \
--lora-name my_lora \
--lora-scale 1.0 \
--output output.png
```
## LoRA Format
LoRA adapters should be in PEFT format, for example:
```
lora_adapter/
├── adapter_config.json
└── adapter_model.safetensors
```
@@ -1,124 +0,0 @@
#!/usr/bin/env python3
"""
OpenAI-compatible chat client for diffusion LoRA inference.
Example:
python openai_chat_client.py \
--server http://localhost:8091 \
--prompt "A piece of cheesecake" \
--lora-path /path/to/lora_adapter \
--lora-name my_lora \
--lora-scale 1.0 \
--output output.png
"""
import argparse
import base64
from pathlib import Path
import requests
def generate_image(
*,
prompt: str,
server_url: str,
height: int | None,
width: int | None,
num_inference_steps: int | None,
seed: int | None,
lora_name: str | None,
lora_path: str | None,
lora_scale: float | None,
lora_int_id: int | None,
) -> bytes | None:
messages = [{"role": "user", "content": prompt}]
extra_body: dict = {}
if height is not None:
extra_body["height"] = height
if width is not None:
extra_body["width"] = width
if num_inference_steps is not None:
extra_body["num_inference_steps"] = num_inference_steps
if seed is not None:
extra_body["seed"] = seed
if lora_path:
lora_body: dict = {
"local_path": lora_path,
"name": lora_name or Path(lora_path).stem,
}
if lora_scale is not None:
lora_body["scale"] = float(lora_scale)
if lora_int_id is not None:
lora_body["int_id"] = int(lora_int_id)
extra_body["lora"] = lora_body
payload = {"messages": messages}
if extra_body:
payload["extra_body"] = extra_body
response = requests.post(
f"{server_url}/v1/chat/completions",
headers={"Content-Type": "application/json"},
json=payload,
timeout=300,
)
response.raise_for_status()
data = response.json()
content = data["choices"][0]["message"]["content"]
if isinstance(content, list) and content:
image_url = content[0].get("image_url", {}).get("url", "")
if image_url.startswith("data:image"):
_, b64_data = image_url.split(",", 1)
return base64.b64decode(b64_data)
raise RuntimeError(f"Unexpected response format: {content!r}")
def main() -> None:
parser = argparse.ArgumentParser(description="Diffusion LoRA OpenAI chat client")
parser.add_argument("--server", default="http://localhost:8091", help="Server URL")
parser.add_argument("--prompt", default="A piece of cheesecake", help="Text prompt")
parser.add_argument("--output", default="lora_online_output.png", help="Output image path")
parser.add_argument("--height", type=int, default=1024, help="Image height")
parser.add_argument("--width", type=int, default=1024, help="Image width")
parser.add_argument("--steps", type=int, default=50, help="num_inference_steps")
parser.add_argument("--seed", type=int, default=42, help="Random seed")
parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)")
parser.add_argument("--lora-name", default=None, help="LoRA name (optional)")
parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale")
parser.add_argument(
"--lora-int-id",
type=int,
default=None,
help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
)
args = parser.parse_args()
image_bytes = generate_image(
prompt=args.prompt,
server_url=args.server,
height=args.height,
width=args.width,
num_inference_steps=args.steps,
seed=args.seed,
lora_name=args.lora_name,
lora_path=args.lora_path,
lora_scale=args.lora_scale if args.lora_path else None,
lora_int_id=args.lora_int_id if args.lora_path else None,
)
out_path = Path(args.output)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_bytes(image_bytes)
print(f"Saved: {out_path} ({len(image_bytes) / 1024:.1f} KiB)")
if __name__ == "__main__":
main()
@@ -1,61 +0,0 @@
#!/bin/bash
# Online diffusion LoRA inference via OpenAI-compatible chat API.
SERVER="${SERVER:-http://localhost:8091}"
PROMPT="${PROMPT:-A piece of cheesecake}"
LORA_PATH="${LORA_PATH:-}"
LORA_NAME="${LORA_NAME:-lora}"
LORA_SCALE="${LORA_SCALE:-1.0}"
LORA_INT_ID="${LORA_INT_ID:-}"
HEIGHT="${HEIGHT:-1024}"
WIDTH="${WIDTH:-1024}"
NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-50}"
SEED="${SEED:-42}"
CURRENT_TIME=$(date +%Y%m%d%H%M%S)
OUTPUT="${OUTPUT:-lora_online_output_${CURRENT_TIME}.png}"
if [ -z "$LORA_PATH" ]; then
echo "ERROR: LORA_PATH is required (must be a server-local path)."
exit 1
fi
echo "Generating image with LoRA..."
echo "Server: $SERVER"
echo "Prompt: $PROMPT"
echo "LoRA: name=$LORA_NAME id=${LORA_INT_ID:-auto} scale=$LORA_SCALE path=$LORA_PATH"
echo "Output: $OUTPUT"
LORA_INT_ID_FIELD=""
if [ -n "$LORA_INT_ID" ]; then
LORA_INT_ID_FIELD=", \"int_id\": $LORA_INT_ID"
fi
curl -s "$SERVER/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"messages\": [
{\"role\": \"user\", \"content\": \"$PROMPT\"}
],
\"extra_body\": {
\"height\": $HEIGHT,
\"width\": $WIDTH,
\"num_inference_steps\": $NUM_INFERENCE_STEPS,
\"seed\": $SEED,
\"lora\": {
\"name\": \"$LORA_NAME\",
\"local_path\": \"$LORA_PATH\",
\"scale\": $LORA_SCALE$LORA_INT_ID_FIELD
}
}
}" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT"
if [ -f "$OUTPUT" ]; then
echo "Image saved to: $OUTPUT"
echo "Size: $(du -h "$OUTPUT" | cut -f1)"
else
echo "Failed to generate image"
exit 1
fi
@@ -1,20 +0,0 @@
#!/bin/bash
# Online diffusion serving with vLLM-Omni (OpenAI-compatible API).
MODEL="${MODEL:-stabilityai/stable-diffusion-3.5-medium}"
PORT="${PORT:-8091}"
echo "Starting vLLM-Omni diffusion server..."
echo "Model: $MODEL"
echo "Port: $PORT"
if [ -z "${VLLM_BIN:-}" ]; then
if command -v vllm-omni >/dev/null 2>&1; then
VLLM_BIN="vllm-omni"
else
VLLM_BIN="vllm"
fi
fi
"$VLLM_BIN" serve "$MODEL" --omni \
--port "$PORT"
@@ -58,6 +58,59 @@ python gradio_demo.py
# Visit http://localhost:7860
```
## LoRA
This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
### Using Python Client with LoRA
```bash
python openai_chat_client.py \
--prompt "A piece of cheesecake" \
--lora-path /path/to/lora_adapter \
--lora-name my_lora \
--lora-scale 1.0 \
--output output.png
```
### Using curl with LoRA (Images API)
The `/v1/images/generations` endpoint supports a `lora` field in the request body:
```bash
curl -X POST http://localhost:8091/v1/images/generations \
-H "Content-Type: application/json" \
-d '{
"prompt": "A piece of cheesecake",
"size": "1024x1024",
"seed": 42,
"lora": {
"name": "my_lora",
"local_path": "/path/to/lora_adapter",
"scale": 1.0
}
}' | jq -r '.data[0].b64_json' | base64 -d > output.png
```
### LoRA Parameters
| Parameter | Type | Description |
| ------------- | ------ | ---------------------------------------------------------------- |
| `name` | str | LoRA adapter name (optional, defaults to path stem) |
| `local_path` | str | Server-local path to LoRA adapter folder (PEFT format, required) |
| `scale` | float | LoRA scale factor (default: 1.0) |
| `int_id` | int | LoRA integer ID for caching (optional, derived from path if not provided) |
### LoRA Adapter Format
LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure:
```
lora_adapter/
├── adapter_config.json
└── adapter_model.safetensors
```
## Request Format
### Simple Text Generation
@@ -24,6 +24,10 @@ def generate_image(
seed: int | None = None,
negative_prompt: str | None = None,
num_outputs_per_prompt: int = 1,
lora_path: str | None = None,
lora_name: str | None = None,
lora_scale: float | None = None,
lora_int_id: int | None = None,
) -> bytes | None:
"""Generate an image using the images generation API.
@@ -37,6 +41,10 @@ def generate_image(
seed: Random seed
negative_prompt: Negative prompt
num_outputs_per_prompt: Number of images to generate
lora_path: Server-local LoRA adapter folder path (PEFT format)
lora_name: LoRA name (optional, defaults to path stem)
lora_scale: LoRA scale factor (default: 1.0)
lora_int_id: LoRA integer ID (optional, derived from path if not provided)
Returns:
Image bytes or None if failed
@@ -63,6 +71,18 @@ def generate_image(
if seed is not None:
payload["seed"] = seed
# Add LoRA if provided
if lora_path:
lora_body: dict = {
"local_path": lora_path,
"name": lora_name or Path(lora_path).stem,
}
if lora_scale is not None:
lora_body["scale"] = float(lora_scale)
if lora_int_id is not None:
lora_body["int_id"] = int(lora_int_id)
payload["lora"] = lora_body
try:
response = requests.post(
f"{server_url}/v1/images/generations",
@@ -99,6 +119,16 @@ def main():
parser.add_argument("--seed", type=int, default=0, help="Random seed")
parser.add_argument("--negative", help="Negative prompt")
parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)")
parser.add_argument("--lora-name", default=None, help="LoRA name (optional)")
parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale")
parser.add_argument(
"--lora-int-id",
type=int,
default=None,
help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
)
args = parser.parse_args()
print(f"Generating image for: {args.prompt}")
@@ -112,6 +142,10 @@ def main():
true_cfg_scale=args.cfg_scale,
seed=args.seed,
negative_prompt=args.negative,
lora_path=args.lora_path,
lora_name=args.lora_name,
lora_scale=args.lora_scale if args.lora_path else None,
lora_int_id=args.lora_int_id if args.lora_path else None,
)
if image_bytes: