95 lines
4.0 KiB
Python
95 lines
4.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
|
|
from vllm_omni.diffusion.data import DiffusionOutput
|
|
from vllm_omni.diffusion.models.dmd2.config import DMD2Config
|
|
from vllm_omni.diffusion.models.schedulers import DMD2EulerScheduler
|
|
from vllm_omni.diffusion.models.utils import _load_json
|
|
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DMD2PipelineMixin:
|
|
"""Mixin for FastGen DMD2-distilled models. Must appear before the base pipeline in MRO."""
|
|
|
|
def __init_dmd2__(self) -> None:
|
|
"""Call after super().__init__() to apply DMD2 scheduler and read model_index."""
|
|
local_files_only = os.path.exists(self.od_config.model)
|
|
try:
|
|
model_index = _load_json(self.od_config.model, "model_index.json", local_files_only)
|
|
except Exception:
|
|
model_index = {}
|
|
|
|
self.dmd2_config = DMD2Config.from_model_index(model_index)
|
|
|
|
self.scheduler = DMD2EulerScheduler(
|
|
num_train_timesteps=1000,
|
|
shift=1.0,
|
|
dmd2_timesteps=self.dmd2_config.resolve_timesteps(),
|
|
stochastic_sampling=(self.dmd2_config.solver == "sde"),
|
|
)
|
|
|
|
def _sanitize_dmd2_request(self, req) -> None:
|
|
"""Sanitize CFG-related fields in-place. Works with both OmniDiffusionRequest and DiffusionRequestBatch."""
|
|
sampling_params = req.sampling_params_list if isinstance(req, DiffusionRequestBatch) else [req.sampling_params]
|
|
for sp in sampling_params:
|
|
if sp.num_inference_steps and sp.num_inference_steps != self.dmd2_config.num_inference_steps:
|
|
logger.warning(
|
|
"DMD2: ignoring num_inference_steps=%d, forcing %d.",
|
|
sp.num_inference_steps,
|
|
self.dmd2_config.num_inference_steps,
|
|
)
|
|
sp.num_inference_steps = self.dmd2_config.num_inference_steps
|
|
|
|
if sp.guidance_scale_provided and sp.guidance_scale != self.dmd2_config.guidance_scale:
|
|
logger.warning(
|
|
"DMD2: ignoring guidance_scale=%.2f, forcing %.2f.",
|
|
sp.guidance_scale,
|
|
self.dmd2_config.guidance_scale,
|
|
)
|
|
sp.guidance_scale = self.dmd2_config.guidance_scale
|
|
sp.guidance_scale_provided = False
|
|
|
|
if sp.guidance_scale_2 is not None:
|
|
logger.warning("DMD2: ignoring guidance_scale_2.")
|
|
sp.guidance_scale_2 = None
|
|
|
|
if sp.true_cfg_scale is not None:
|
|
logger.warning("DMD2: ignoring true_cfg_scale.")
|
|
sp.true_cfg_scale = None
|
|
|
|
sp.do_classifier_free_guidance = False
|
|
sp.is_cfg_negative = False
|
|
|
|
# Prevent base pipelines from replacing the DMD2 scheduler.
|
|
extra_args = getattr(sp, "extra_args", None) or {}
|
|
for key in ("sample_solver", "flow_shift"):
|
|
if key in extra_args:
|
|
logger.warning("DMD2: ignoring extra_args.%s.", key)
|
|
extra_args.pop(key)
|
|
|
|
# Strip negative_prompt from each request's prompt in-place.
|
|
requests = req.requests if hasattr(req, "requests") else [req]
|
|
for request in requests:
|
|
p = request.prompt
|
|
if isinstance(p, dict) and "negative_prompt" in p:
|
|
logger.warning("DMD2: ignoring negative_prompt.")
|
|
request.prompt = {k: v for k, v in p.items() if k != "negative_prompt"}
|
|
|
|
def forward(self, req: DiffusionRequestBatch, **kwargs) -> list[DiffusionOutput]:
|
|
self._sanitize_dmd2_request(req)
|
|
kwargs.pop("guidance_scale", None)
|
|
kwargs.pop("num_inference_steps", None)
|
|
return super().forward(
|
|
req,
|
|
guidance_scale=self.dmd2_config.guidance_scale,
|
|
num_inference_steps=self.dmd2_config.num_inference_steps,
|
|
**kwargs,
|
|
)
|