[Feature]: Adds distilled LoRA support for diffusion models (#2783)

Signed-off-by: Songrui625 <songrui625@gmail.com>
This commit is contained in:
Jerry Song authored and GitHub committed 2026-08-11 14:23:43 +00:00
1 parent 348cf95a89
commit 89015d5cad
24 files changed
+1283 -33

No files matched your search

@@ -60,6 +60,8 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
from vllm_omni.model_extras import (
build_image_to_video_prompt,
get_extra_body_params,
@@ -294,6 +296,30 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
parser.add_argument(
"--lora-path",
type=str,
nargs="+",
default=None,
help="Path to a LoRA adapter folder (PEFT format) or concrete LoRA checkpoint files. "
"For Wan2.2 MoE models, pass the high-noise checkpoint first and the low-noise checkpoint second.",
)
parser.add_argument(
"--lora-scale",
type=float,
default=1.0,
help="Scale factor for PEFT LoRA weights (default: 1.0).",
)
parser.add_argument(
"--lora-backend",
type=str,
default="peft",
choices=["peft", "distill"],
help=(
"LoRA backend. 'peft' loads a PEFT-format adapter for request-time activation; "
"'distill' fuses one or more concrete LoRA checkpoint files at initialization."
),
)
parser.add_argument(
"--extra-body",
type=functools.partial(parse_json_object, flag_name="--extra-body"),
@@ -487,6 +513,12 @@ def main():
omni_kwargs["flow_shift"] = flow_shift
if args.quantization is not None:
omni_kwargs["quantization"] = args.quantization
if args.lora_path is not None:
lora_path = args.lora_path
if len(lora_path) == 1:
lora_path = lora_path[0]
omni_kwargs["lora_path"] = lora_path
omni_kwargs["lora_backend"] = args.lora_backend
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
# gate is an engine-level config (offline analog of the server's --no-guardrails).
if args.extra_body and "guardrails" in args.extra_body:
@@ -517,6 +549,18 @@ def main():
print(f" Video size: {width}x{height}")
print(f"{'=' * 60}\n")
lora_request = None
if args.lora_path and args.lora_backend == "peft":
if len(args.lora_path) != 1:
raise ValueError("Only one LoRA path is expected for PEFT backend.")
lora_path = args.lora_path[0]
lora_request = LoRARequest(
lora_name=Path(lora_path).stem,
lora_int_id=stable_lora_int_id(lora_path),
lora_path=lora_path,
)
negative_prompt = args.negative_prompt
if negative_prompt is None and not is_ltx2:
# Preserve the historical empty-prompt behavior for non-LTX examples.
@@ -530,6 +574,11 @@ def main():
width=width,
num_frames=num_frames,
)
sampling_extra_args = {"sample_solver": args.sample_solver}
if lora_request is not None:
sampling_extra_args["lora_request"] = lora_request
sampling_extra_args["lora_scale"] = args.lora_scale
sampling_params = OmniDiffusionSamplingParams(
height=height,
width=width,
@@ -540,7 +589,7 @@ def main():
num_inference_steps=num_inference_steps,
num_frames=num_frames,
frame_rate=frame_rate,
extra_args={"sample_solver": args.sample_solver},
extra_args=sampling_extra_args,
)
if flow_shift is not None:
sampling_params.extra_args["flow_shift"] = flow_shift
@@ -109,8 +109,9 @@ python text_to_image.py \
| `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference |
| `--ulysses-mode` | str | `"strict"` | Ulysses SP mode: `"strict"` or `"advanced_uaa"` |
| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder |
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
| `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
| `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
@@ -260,6 +260,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--lora-path",
type=str,
nargs="+",
default=None,
help="Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation.",
)
@@ -269,6 +270,15 @@ def parse_args() -> argparse.Namespace:
default=1.0,
help="Scale factor for LoRA weights (default: 1.0).",
)
parser.add_argument(
"--lora-backend",
type=str,
default="peft",
choices=["peft", "distill"],
help="LoRA backend for loading LoRA adapters. Default: peft"
"'peft' loads a PEFT-format adapter folder, used e.g. for RL"
"'distill' fuses one or more concrete LoRA checkpoint files, used e.g. for distilled few-step LoRAs",
)
parser.add_argument(
"--vae-patch-parallel-size",
type=int,
@@ -404,8 +414,12 @@ def main():
# Prepare LoRA kwargs for Omni initialization
lora_args: dict[str, Any] = {}
if args.lora_path:
lora_args["lora_path"] = args.lora_path
print(f"Using LoRA from: {args.lora_path}")
lora_path = args.lora_path
if len(lora_path) == 1:
lora_path = lora_path[0]
lora_args["lora_path"] = lora_path
lora_args["lora_backend"] = args.lora_backend
print(f"Using LoRA from: {lora_path} with backend: {args.lora_backend}")
# Build quantization kwargs: use quantization_config dict when
# ignored_layers is specified so the list flows through OmniDiffusionConfig
@@ -494,12 +508,16 @@ def main():
# Build LoRA request when --lora-path is set
lora_request = None
if args.lora_path:
lora_request_id = stable_lora_int_id(args.lora_path)
if args.lora_path and args.lora_backend == "peft":
if len(args.lora_path) != 1:
raise ValueError("Only one LoRA path is expected for PEFT backend.")
lora_path = args.lora_path[0]
lora_request_id = stable_lora_int_id(lora_path)
lora_request = LoRARequest(
lora_name=Path(args.lora_path).stem,
lora_name=Path(lora_path).stem,
lora_int_id=lora_request_id,
lora_path=args.lora_path,
lora_path=lora_path,
)
generation_start = time.perf_counter()
@@ -218,6 +218,9 @@ python text_to_video.py \
- `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
- `--quantization`: quantization method (such as `fp8` for FP8).
- `--flow-shift`: scheduler flow_shift parameter.
- `--lora-path`: path to PEFT LoRA adapter folder or checkpoint file.
- `--lora-scale`: scale factor for LoRA weights.
- `--lora-backend`: backend for loading LoRA adapters. Default: peft. Available options: peft, distill.
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3 (see above).
### Wan2.2-specific
@@ -14,6 +14,8 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -334,6 +336,29 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Enable expert parallelism for MoE layers.",
)
parser.add_argument(
"--lora-path",
type=str,
nargs="+",
default=None,
help="Path to LoRA adapter folder (PEFT format) or concrete LoRA checkpoint files. Loaded at initialization and used for generation."
"Note: for Wan2.2 MoE models, two checkpoints positionally mapped to high-noise and low-noise modules.",
)
parser.add_argument(
"--lora-scale",
type=float,
default=1.0,
help="Scale factor for LoRA weights (default: 1.0).",
)
parser.add_argument(
"--lora-backend",
type=str,
default="peft",
choices=["peft", "distill"],
help="LoRA backend for loading LoRA adapters. Default: peft"
"'peft' loads a PEFT-format adapter folder, used e.g. for RL"
"'distill' fuses one or more concrete LoRA checkpoint files, used e.g. for distilled few-step LoRAs",
)
parser.add_argument(
"--use-hsdp",
action="store_true",
@@ -440,6 +465,12 @@ def main():
omni_kwargs["cache_backend"] = args.cache_backend
omni_kwargs["cache_config"] = cache_config
omni_kwargs["enable_cache_dit_summary"] = args.enable_cache_dit_summary
if args.lora_path is not None:
lora_path = args.lora_path
if len(lora_path) == 1:
lora_path = lora_path[0]
omni_kwargs["lora_path"] = lora_path
omni_kwargs["lora_backend"] = args.lora_backend
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
# gate is an engine-level config (offline analog of the server's --no-guardrails).
@@ -469,6 +500,19 @@ def main():
print(f" Video size: {args.width}x{args.height}")
print(f"{'=' * 60}\n")
lora_request = None
if args.lora_path and args.lora_backend == "peft":
if len(args.lora_path) != 1:
raise ValueError("Only one LoRA path is expected for PEFT backend.")
lora_path = args.lora_path[0]
lora_request_id = stable_lora_int_id(lora_path)
lora_request = LoRARequest(
lora_name=Path(lora_path).stem,
lora_int_id=lora_request_id,
lora_path=lora_path,
)
prompt_dict = {"prompt": args.prompt}
if args.negative_prompt is not None:
prompt_dict["negative_prompt"] = args.negative_prompt
@@ -476,6 +520,11 @@ def main():
# Preserve the historical empty-prompt behavior for non-LTX examples.
prompt_dict["negative_prompt"] = ""
extra_args = {}
if lora_request:
extra_args["lora_request"] = lora_request
extra_args["lora_scale"] = args.lora_scale
sampling_kwargs = dict(
height=args.height,
width=args.width,
@@ -483,6 +532,7 @@ def main():
guidance_scale=args.guidance_scale,
num_inference_steps=args.num_inference_steps,
num_frames=args.num_frames,
extra_args=extra_args,
)
if args.guidance_scale_high is not None:
sampling_kwargs["guidance_scale_2"] = args.guidance_scale_high
@@ -115,7 +115,23 @@ python gradio_demo.py
## LoRA
This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
This example supports both startup-time distilled LoRAs and request-time
PEFT-compatible adapters. The LoRA path must be readable on the **server**
machine (usually a local path or a mounted directory).
### Start Server with a Distilled LoRA
```bash
vllm serve Qwen/Qwen-Image-2512 \
--omni \
--port 8091 \
--lora-backend distill \
--lora-path /path/to/Qwen-Image-2512-Lightning-4steps.safetensors
```
The distilled LoRA is fused once during server initialization and applies to
every request. Use the checkpoint's few-step sampling settings, for example
`num_inference_steps=4` and `true_cfg_scale=1.0`.
### Using Python Client with LoRA