[Feature]: Adds distilled LoRA support for diffusion models (#2783)
Signed-off-by: Songrui625 <songrui625@gmail.com>
This commit is contained in:
1 parent
348cf95a89
commit
89015d5cad
24 files changed
+1283
-33
No files matched your search
@@ -60,6 +60,8 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
from vllm_omni.model_extras import (
|
||||
build_image_to_video_prompt,
|
||||
get_extra_body_params,
|
||||
@@ -294,6 +296,30 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-path",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Path to a LoRA adapter folder (PEFT format) or concrete LoRA checkpoint files. "
|
||||
"For Wan2.2 MoE models, pass the high-noise checkpoint first and the low-noise checkpoint second.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Scale factor for PEFT LoRA weights (default: 1.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-backend",
|
||||
type=str,
|
||||
default="peft",
|
||||
choices=["peft", "distill"],
|
||||
help=(
|
||||
"LoRA backend. 'peft' loads a PEFT-format adapter for request-time activation; "
|
||||
"'distill' fuses one or more concrete LoRA checkpoint files at initialization."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=functools.partial(parse_json_object, flag_name="--extra-body"),
|
||||
@@ -487,6 +513,12 @@ def main():
|
||||
omni_kwargs["flow_shift"] = flow_shift
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
if args.lora_path is not None:
|
||||
lora_path = args.lora_path
|
||||
if len(lora_path) == 1:
|
||||
lora_path = lora_path[0]
|
||||
omni_kwargs["lora_path"] = lora_path
|
||||
omni_kwargs["lora_backend"] = args.lora_backend
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
@@ -517,6 +549,18 @@ def main():
|
||||
print(f" Video size: {width}x{height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
lora_request = None
|
||||
if args.lora_path and args.lora_backend == "peft":
|
||||
if len(args.lora_path) != 1:
|
||||
raise ValueError("Only one LoRA path is expected for PEFT backend.")
|
||||
|
||||
lora_path = args.lora_path[0]
|
||||
lora_request = LoRARequest(
|
||||
lora_name=Path(lora_path).stem,
|
||||
lora_int_id=stable_lora_int_id(lora_path),
|
||||
lora_path=lora_path,
|
||||
)
|
||||
|
||||
negative_prompt = args.negative_prompt
|
||||
if negative_prompt is None and not is_ltx2:
|
||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||
@@ -530,6 +574,11 @@ def main():
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
sampling_extra_args = {"sample_solver": args.sample_solver}
|
||||
if lora_request is not None:
|
||||
sampling_extra_args["lora_request"] = lora_request
|
||||
sampling_extra_args["lora_scale"] = args.lora_scale
|
||||
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
@@ -540,7 +589,7 @@ def main():
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={"sample_solver": args.sample_solver},
|
||||
extra_args=sampling_extra_args,
|
||||
)
|
||||
if flow_shift is not None:
|
||||
sampling_params.extra_args["flow_shift"] = flow_shift
|
||||
|
||||
@@ -109,8 +109,9 @@ python text_to_image.py \
|
||||
| `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference |
|
||||
| `--ulysses-mode` | str | `"strict"` | Ulysses SP mode: `"strict"` or `"advanced_uaa"` |
|
||||
| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
|
||||
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder |
|
||||
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
|
||||
| `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
|
||||
| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
|
||||
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
|
||||
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
|
||||
| `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
|
||||
|
||||
@@ -260,6 +260,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--lora-path",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation.",
|
||||
)
|
||||
@@ -269,6 +270,15 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1.0,
|
||||
help="Scale factor for LoRA weights (default: 1.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-backend",
|
||||
type=str,
|
||||
default="peft",
|
||||
choices=["peft", "distill"],
|
||||
help="LoRA backend for loading LoRA adapters. Default: peft"
|
||||
"'peft' loads a PEFT-format adapter folder, used e.g. for RL"
|
||||
"'distill' fuses one or more concrete LoRA checkpoint files, used e.g. for distilled few-step LoRAs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae-patch-parallel-size",
|
||||
type=int,
|
||||
@@ -404,8 +414,12 @@ def main():
|
||||
# Prepare LoRA kwargs for Omni initialization
|
||||
lora_args: dict[str, Any] = {}
|
||||
if args.lora_path:
|
||||
lora_args["lora_path"] = args.lora_path
|
||||
print(f"Using LoRA from: {args.lora_path}")
|
||||
lora_path = args.lora_path
|
||||
if len(lora_path) == 1:
|
||||
lora_path = lora_path[0]
|
||||
lora_args["lora_path"] = lora_path
|
||||
lora_args["lora_backend"] = args.lora_backend
|
||||
print(f"Using LoRA from: {lora_path} with backend: {args.lora_backend}")
|
||||
|
||||
# Build quantization kwargs: use quantization_config dict when
|
||||
# ignored_layers is specified so the list flows through OmniDiffusionConfig
|
||||
@@ -494,12 +508,16 @@ def main():
|
||||
|
||||
# Build LoRA request when --lora-path is set
|
||||
lora_request = None
|
||||
if args.lora_path:
|
||||
lora_request_id = stable_lora_int_id(args.lora_path)
|
||||
if args.lora_path and args.lora_backend == "peft":
|
||||
if len(args.lora_path) != 1:
|
||||
raise ValueError("Only one LoRA path is expected for PEFT backend.")
|
||||
|
||||
lora_path = args.lora_path[0]
|
||||
lora_request_id = stable_lora_int_id(lora_path)
|
||||
lora_request = LoRARequest(
|
||||
lora_name=Path(args.lora_path).stem,
|
||||
lora_name=Path(lora_path).stem,
|
||||
lora_int_id=lora_request_id,
|
||||
lora_path=args.lora_path,
|
||||
lora_path=lora_path,
|
||||
)
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
|
||||
@@ -218,6 +218,9 @@ python text_to_video.py \
|
||||
- `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
|
||||
- `--quantization`: quantization method (such as `fp8` for FP8).
|
||||
- `--flow-shift`: scheduler flow_shift parameter.
|
||||
- `--lora-path`: path to PEFT LoRA adapter folder or checkpoint file.
|
||||
- `--lora-scale`: scale factor for LoRA weights.
|
||||
- `--lora-backend`: backend for loading LoRA adapters. Default: peft. Available options: peft, distill.
|
||||
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3 (see above).
|
||||
|
||||
### Wan2.2-specific
|
||||
|
||||
@@ -14,6 +14,8 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -334,6 +336,29 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable expert parallelism for MoE layers.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-path",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Path to LoRA adapter folder (PEFT format) or concrete LoRA checkpoint files. Loaded at initialization and used for generation."
|
||||
"Note: for Wan2.2 MoE models, two checkpoints positionally mapped to high-noise and low-noise modules.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Scale factor for LoRA weights (default: 1.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-backend",
|
||||
type=str,
|
||||
default="peft",
|
||||
choices=["peft", "distill"],
|
||||
help="LoRA backend for loading LoRA adapters. Default: peft"
|
||||
"'peft' loads a PEFT-format adapter folder, used e.g. for RL"
|
||||
"'distill' fuses one or more concrete LoRA checkpoint files, used e.g. for distilled few-step LoRAs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-hsdp",
|
||||
action="store_true",
|
||||
@@ -440,6 +465,12 @@ def main():
|
||||
omni_kwargs["cache_backend"] = args.cache_backend
|
||||
omni_kwargs["cache_config"] = cache_config
|
||||
omni_kwargs["enable_cache_dit_summary"] = args.enable_cache_dit_summary
|
||||
if args.lora_path is not None:
|
||||
lora_path = args.lora_path
|
||||
if len(lora_path) == 1:
|
||||
lora_path = lora_path[0]
|
||||
omni_kwargs["lora_path"] = lora_path
|
||||
omni_kwargs["lora_backend"] = args.lora_backend
|
||||
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
@@ -469,6 +500,19 @@ def main():
|
||||
print(f" Video size: {args.width}x{args.height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
lora_request = None
|
||||
if args.lora_path and args.lora_backend == "peft":
|
||||
if len(args.lora_path) != 1:
|
||||
raise ValueError("Only one LoRA path is expected for PEFT backend.")
|
||||
|
||||
lora_path = args.lora_path[0]
|
||||
lora_request_id = stable_lora_int_id(lora_path)
|
||||
lora_request = LoRARequest(
|
||||
lora_name=Path(lora_path).stem,
|
||||
lora_int_id=lora_request_id,
|
||||
lora_path=lora_path,
|
||||
)
|
||||
|
||||
prompt_dict = {"prompt": args.prompt}
|
||||
if args.negative_prompt is not None:
|
||||
prompt_dict["negative_prompt"] = args.negative_prompt
|
||||
@@ -476,6 +520,11 @@ def main():
|
||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||
prompt_dict["negative_prompt"] = ""
|
||||
|
||||
extra_args = {}
|
||||
if lora_request:
|
||||
extra_args["lora_request"] = lora_request
|
||||
extra_args["lora_scale"] = args.lora_scale
|
||||
|
||||
sampling_kwargs = dict(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
@@ -483,6 +532,7 @@ def main():
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_frames=args.num_frames,
|
||||
extra_args=extra_args,
|
||||
)
|
||||
if args.guidance_scale_high is not None:
|
||||
sampling_kwargs["guidance_scale_2"] = args.guidance_scale_high
|
||||
|
||||
@@ -115,7 +115,23 @@ python gradio_demo.py
|
||||
|
||||
## LoRA
|
||||
|
||||
This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
|
||||
This example supports both startup-time distilled LoRAs and request-time
|
||||
PEFT-compatible adapters. The LoRA path must be readable on the **server**
|
||||
machine (usually a local path or a mounted directory).
|
||||
|
||||
### Start Server with a Distilled LoRA
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen-Image-2512 \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--lora-backend distill \
|
||||
--lora-path /path/to/Qwen-Image-2512-Lightning-4steps.safetensors
|
||||
```
|
||||
|
||||
The distilled LoRA is fused once during server initialization and applies to
|
||||
every request. Use the checkpoint's few-step sampling settings, for example
|
||||
`num_inference_steps=4` and `true_cfg_scale=1.0`.
|
||||
|
||||
### Using Python Client with LoRA
|
||||
|
||||
|
||||
Reference in new issue
Block a user