[Doc] format lora related docs for the user's end (#1009)
Signed-off-by: AndyZhou952 <jzhoubc@connect.ust.hk> Signed-off-by: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com>
This commit is contained in:
1 parent
caed95fb45
commit
a2d5b61703
16 files changed
+314
-687
No files matched your search
@@ -1,54 +0,0 @@
|
||||
# LoRA Inference(Diffusion)
|
||||
|
||||
This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API.
|
||||
|
||||
> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
|
||||
> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here.
|
||||
|
||||
## Start Server
|
||||
|
||||
```bash
|
||||
# Pick a diffusion model (examples)
|
||||
# export MODEL=stabilityai/stable-diffusion-3.5-medium
|
||||
# export MODEL=Qwen/Qwen-Image
|
||||
|
||||
bash run_server.sh
|
||||
```
|
||||
|
||||
## Call API (curl)
|
||||
|
||||
```bash
|
||||
# Required: local LoRA folder on the server
|
||||
export LORA_PATH=/path/to/lora_adapter
|
||||
|
||||
# Optional
|
||||
export SERVER=http://localhost:8091
|
||||
export PROMPT="A piece of cheesecake"
|
||||
export LORA_NAME=my_lora
|
||||
export LORA_SCALE=1.0
|
||||
# Optional: if omitted, the server derives a stable id from LORA_PATH.
|
||||
# export LORA_INT_ID=123
|
||||
|
||||
bash run_curl_lora_inference.sh
|
||||
```
|
||||
|
||||
## Call API (Python)
|
||||
|
||||
```bash
|
||||
python openai_chat_client.py \
|
||||
--prompt "A piece of cheesecake" \
|
||||
--lora-path /path/to/lora_adapter \
|
||||
--lora-name my_lora \
|
||||
--lora-scale 1.0 \
|
||||
--output output.png
|
||||
```
|
||||
|
||||
## LoRA Format
|
||||
|
||||
LoRA adapters should be in PEFT format, for example:
|
||||
|
||||
```
|
||||
lora_adapter/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
```
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OpenAI-compatible chat client for diffusion LoRA inference.
|
||||
|
||||
Example:
|
||||
python openai_chat_client.py \
|
||||
--server http://localhost:8091 \
|
||||
--prompt "A piece of cheesecake" \
|
||||
--lora-path /path/to/lora_adapter \
|
||||
--lora-name my_lora \
|
||||
--lora-scale 1.0 \
|
||||
--output output.png
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def generate_image(
|
||||
*,
|
||||
prompt: str,
|
||||
server_url: str,
|
||||
height: int | None,
|
||||
width: int | None,
|
||||
num_inference_steps: int | None,
|
||||
seed: int | None,
|
||||
lora_name: str | None,
|
||||
lora_path: str | None,
|
||||
lora_scale: float | None,
|
||||
lora_int_id: int | None,
|
||||
) -> bytes | None:
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
extra_body: dict = {}
|
||||
if height is not None:
|
||||
extra_body["height"] = height
|
||||
if width is not None:
|
||||
extra_body["width"] = width
|
||||
if num_inference_steps is not None:
|
||||
extra_body["num_inference_steps"] = num_inference_steps
|
||||
if seed is not None:
|
||||
extra_body["seed"] = seed
|
||||
|
||||
if lora_path:
|
||||
lora_body: dict = {
|
||||
"local_path": lora_path,
|
||||
"name": lora_name or Path(lora_path).stem,
|
||||
}
|
||||
if lora_scale is not None:
|
||||
lora_body["scale"] = float(lora_scale)
|
||||
if lora_int_id is not None:
|
||||
lora_body["int_id"] = int(lora_int_id)
|
||||
extra_body["lora"] = lora_body
|
||||
|
||||
payload = {"messages": messages}
|
||||
if extra_body:
|
||||
payload["extra_body"] = extra_body
|
||||
|
||||
response = requests.post(
|
||||
f"{server_url}/v1/chat/completions",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=300,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
if isinstance(content, list) and content:
|
||||
image_url = content[0].get("image_url", {}).get("url", "")
|
||||
if image_url.startswith("data:image"):
|
||||
_, b64_data = image_url.split(",", 1)
|
||||
return base64.b64decode(b64_data)
|
||||
|
||||
raise RuntimeError(f"Unexpected response format: {content!r}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Diffusion LoRA OpenAI chat client")
|
||||
parser.add_argument("--server", default="http://localhost:8091", help="Server URL")
|
||||
parser.add_argument("--prompt", default="A piece of cheesecake", help="Text prompt")
|
||||
parser.add_argument("--output", default="lora_online_output.png", help="Output image path")
|
||||
|
||||
parser.add_argument("--height", type=int, default=1024, help="Image height")
|
||||
parser.add_argument("--width", type=int, default=1024, help="Image width")
|
||||
parser.add_argument("--steps", type=int, default=50, help="num_inference_steps")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed")
|
||||
|
||||
parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)")
|
||||
parser.add_argument("--lora-name", default=None, help="LoRA name (optional)")
|
||||
parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale")
|
||||
parser.add_argument(
|
||||
"--lora-int-id",
|
||||
type=int,
|
||||
default=None,
|
||||
help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
image_bytes = generate_image(
|
||||
prompt=args.prompt,
|
||||
server_url=args.server,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_inference_steps=args.steps,
|
||||
seed=args.seed,
|
||||
lora_name=args.lora_name,
|
||||
lora_path=args.lora_path,
|
||||
lora_scale=args.lora_scale if args.lora_path else None,
|
||||
lora_int_id=args.lora_int_id if args.lora_path else None,
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_bytes(image_bytes)
|
||||
print(f"Saved: {out_path} ({len(image_bytes) / 1024:.1f} KiB)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Online diffusion LoRA inference via OpenAI-compatible chat API.
|
||||
|
||||
SERVER="${SERVER:-http://localhost:8091}"
|
||||
PROMPT="${PROMPT:-A piece of cheesecake}"
|
||||
|
||||
LORA_PATH="${LORA_PATH:-}"
|
||||
LORA_NAME="${LORA_NAME:-lora}"
|
||||
LORA_SCALE="${LORA_SCALE:-1.0}"
|
||||
LORA_INT_ID="${LORA_INT_ID:-}"
|
||||
|
||||
HEIGHT="${HEIGHT:-1024}"
|
||||
WIDTH="${WIDTH:-1024}"
|
||||
NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-50}"
|
||||
SEED="${SEED:-42}"
|
||||
|
||||
CURRENT_TIME=$(date +%Y%m%d%H%M%S)
|
||||
OUTPUT="${OUTPUT:-lora_online_output_${CURRENT_TIME}.png}"
|
||||
|
||||
if [ -z "$LORA_PATH" ]; then
|
||||
echo "ERROR: LORA_PATH is required (must be a server-local path)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Generating image with LoRA..."
|
||||
echo "Server: $SERVER"
|
||||
echo "Prompt: $PROMPT"
|
||||
echo "LoRA: name=$LORA_NAME id=${LORA_INT_ID:-auto} scale=$LORA_SCALE path=$LORA_PATH"
|
||||
echo "Output: $OUTPUT"
|
||||
|
||||
LORA_INT_ID_FIELD=""
|
||||
if [ -n "$LORA_INT_ID" ]; then
|
||||
LORA_INT_ID_FIELD=", \"int_id\": $LORA_INT_ID"
|
||||
fi
|
||||
|
||||
curl -s "$SERVER/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"messages\": [
|
||||
{\"role\": \"user\", \"content\": \"$PROMPT\"}
|
||||
],
|
||||
\"extra_body\": {
|
||||
\"height\": $HEIGHT,
|
||||
\"width\": $WIDTH,
|
||||
\"num_inference_steps\": $NUM_INFERENCE_STEPS,
|
||||
\"seed\": $SEED,
|
||||
\"lora\": {
|
||||
\"name\": \"$LORA_NAME\",
|
||||
\"local_path\": \"$LORA_PATH\",
|
||||
\"scale\": $LORA_SCALE$LORA_INT_ID_FIELD
|
||||
}
|
||||
}
|
||||
}" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT"
|
||||
|
||||
if [ -f "$OUTPUT" ]; then
|
||||
echo "Image saved to: $OUTPUT"
|
||||
echo "Size: $(du -h "$OUTPUT" | cut -f1)"
|
||||
else
|
||||
echo "Failed to generate image"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Online diffusion serving with vLLM-Omni (OpenAI-compatible API).
|
||||
|
||||
MODEL="${MODEL:-stabilityai/stable-diffusion-3.5-medium}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
echo "Starting vLLM-Omni diffusion server..."
|
||||
echo "Model: $MODEL"
|
||||
echo "Port: $PORT"
|
||||
|
||||
if [ -z "${VLLM_BIN:-}" ]; then
|
||||
if command -v vllm-omni >/dev/null 2>&1; then
|
||||
VLLM_BIN="vllm-omni"
|
||||
else
|
||||
VLLM_BIN="vllm"
|
||||
fi
|
||||
fi
|
||||
|
||||
"$VLLM_BIN" serve "$MODEL" --omni \
|
||||
--port "$PORT"
|
||||
@@ -58,6 +58,59 @@ python gradio_demo.py
|
||||
# Visit http://localhost:7860
|
||||
```
|
||||
|
||||
## LoRA
|
||||
|
||||
This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
|
||||
|
||||
### Using Python Client with LoRA
|
||||
|
||||
```bash
|
||||
python openai_chat_client.py \
|
||||
--prompt "A piece of cheesecake" \
|
||||
--lora-path /path/to/lora_adapter \
|
||||
--lora-name my_lora \
|
||||
--lora-scale 1.0 \
|
||||
--output output.png
|
||||
```
|
||||
|
||||
### Using curl with LoRA (Images API)
|
||||
|
||||
The `/v1/images/generations` endpoint supports a `lora` field in the request body:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/images/generations \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"prompt": "A piece of cheesecake",
|
||||
"size": "1024x1024",
|
||||
"seed": 42,
|
||||
"lora": {
|
||||
"name": "my_lora",
|
||||
"local_path": "/path/to/lora_adapter",
|
||||
"scale": 1.0
|
||||
}
|
||||
}' | jq -r '.data[0].b64_json' | base64 -d > output.png
|
||||
```
|
||||
|
||||
### LoRA Parameters
|
||||
|
||||
| Parameter | Type | Description |
|
||||
| ------------- | ------ | ---------------------------------------------------------------- |
|
||||
| `name` | str | LoRA adapter name (optional, defaults to path stem) |
|
||||
| `local_path` | str | Server-local path to LoRA adapter folder (PEFT format, required) |
|
||||
| `scale` | float | LoRA scale factor (default: 1.0) |
|
||||
| `int_id` | int | LoRA integer ID for caching (optional, derived from path if not provided) |
|
||||
|
||||
### LoRA Adapter Format
|
||||
|
||||
LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure:
|
||||
|
||||
```
|
||||
lora_adapter/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
```
|
||||
|
||||
## Request Format
|
||||
|
||||
### Simple Text Generation
|
||||
|
||||
@@ -24,6 +24,10 @@ def generate_image(
|
||||
seed: int | None = None,
|
||||
negative_prompt: str | None = None,
|
||||
num_outputs_per_prompt: int = 1,
|
||||
lora_path: str | None = None,
|
||||
lora_name: str | None = None,
|
||||
lora_scale: float | None = None,
|
||||
lora_int_id: int | None = None,
|
||||
) -> bytes | None:
|
||||
"""Generate an image using the images generation API.
|
||||
|
||||
@@ -37,6 +41,10 @@ def generate_image(
|
||||
seed: Random seed
|
||||
negative_prompt: Negative prompt
|
||||
num_outputs_per_prompt: Number of images to generate
|
||||
lora_path: Server-local LoRA adapter folder path (PEFT format)
|
||||
lora_name: LoRA name (optional, defaults to path stem)
|
||||
lora_scale: LoRA scale factor (default: 1.0)
|
||||
lora_int_id: LoRA integer ID (optional, derived from path if not provided)
|
||||
|
||||
Returns:
|
||||
Image bytes or None if failed
|
||||
@@ -63,6 +71,18 @@ def generate_image(
|
||||
if seed is not None:
|
||||
payload["seed"] = seed
|
||||
|
||||
# Add LoRA if provided
|
||||
if lora_path:
|
||||
lora_body: dict = {
|
||||
"local_path": lora_path,
|
||||
"name": lora_name or Path(lora_path).stem,
|
||||
}
|
||||
if lora_scale is not None:
|
||||
lora_body["scale"] = float(lora_scale)
|
||||
if lora_int_id is not None:
|
||||
lora_body["int_id"] = int(lora_int_id)
|
||||
payload["lora"] = lora_body
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{server_url}/v1/images/generations",
|
||||
@@ -99,6 +119,16 @@ def main():
|
||||
parser.add_argument("--seed", type=int, default=0, help="Random seed")
|
||||
parser.add_argument("--negative", help="Negative prompt")
|
||||
|
||||
parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)")
|
||||
parser.add_argument("--lora-name", default=None, help="LoRA name (optional)")
|
||||
parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale")
|
||||
parser.add_argument(
|
||||
"--lora-int-id",
|
||||
type=int,
|
||||
default=None,
|
||||
help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Generating image for: {args.prompt}")
|
||||
@@ -112,6 +142,10 @@ def main():
|
||||
true_cfg_scale=args.cfg_scale,
|
||||
seed=args.seed,
|
||||
negative_prompt=args.negative,
|
||||
lora_path=args.lora_path,
|
||||
lora_name=args.lora_name,
|
||||
lora_scale=args.lora_scale if args.lora_path else None,
|
||||
lora_int_id=args.lora_int_id if args.lora_path else None,
|
||||
)
|
||||
|
||||
if image_bytes:
|
||||
|
||||
Reference in new issue
Block a user