89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
|
|
from vllm_omni.diffusion.data import DiffusionOutput
|
|
from vllm_omni.diffusion.models.schedulers import DMD2EulerScheduler
|
|
from vllm_omni.diffusion.models.utils import _load_json
|
|
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DMD2PipelineMixin:
|
|
"""Mixin for FastGen DMD2-distilled models. Must appear before the base pipeline in MRO."""
|
|
|
|
def __init_dmd2__(self) -> None:
|
|
"""Call after super().__init__() to apply DMD2 scheduler and read model_index."""
|
|
local_files_only = os.path.exists(self.od_config.model)
|
|
try:
|
|
model_index = _load_json(self.od_config.model, "model_index.json", local_files_only)
|
|
except Exception:
|
|
model_index = {}
|
|
|
|
dmd2_timesteps = model_index.get("dmd2_denoising_timesteps", [999, 937, 833, 624])
|
|
self.num_inference_steps = model_index.get("dmd2_num_inference_steps", 4)
|
|
shift = model_index.get("dmd2_scheduler_shift", 1.0)
|
|
self.dmd2_guidance_scale = model_index.get("dmd2_guidance_scale", 1.0)
|
|
|
|
self.scheduler = DMD2EulerScheduler(
|
|
num_train_timesteps=1000,
|
|
shift=shift,
|
|
dmd2_timesteps=dmd2_timesteps,
|
|
)
|
|
|
|
def _sanitize_dmd2_request(self, req: OmniDiffusionRequest) -> None:
|
|
"""Sanitize CFG-related fields in-place. Mutates req.sampling_params and req.prompts."""
|
|
sp = req.sampling_params
|
|
|
|
if sp.num_inference_steps and sp.num_inference_steps != self.num_inference_steps:
|
|
logger.warning(
|
|
"DMD2: ignoring num_inference_steps=%d, forcing %d.",
|
|
sp.num_inference_steps,
|
|
self.num_inference_steps,
|
|
)
|
|
sp.num_inference_steps = self.num_inference_steps
|
|
|
|
if sp.guidance_scale_provided and sp.guidance_scale != self.dmd2_guidance_scale:
|
|
logger.warning(
|
|
"DMD2: ignoring guidance_scale=%.2f, forcing %.2f.",
|
|
sp.guidance_scale,
|
|
self.dmd2_guidance_scale,
|
|
)
|
|
sp.guidance_scale = self.dmd2_guidance_scale
|
|
sp.guidance_scale_provided = False
|
|
|
|
if sp.guidance_scale_2 is not None:
|
|
logger.warning("DMD2: ignoring guidance_scale_2.")
|
|
sp.guidance_scale_2 = None
|
|
|
|
if sp.true_cfg_scale is not None:
|
|
logger.warning("DMD2: ignoring true_cfg_scale.")
|
|
sp.true_cfg_scale = None
|
|
|
|
sp.do_classifier_free_guidance = False
|
|
sp.is_cfg_negative = False
|
|
|
|
fixed = []
|
|
for p in req.prompts:
|
|
if isinstance(p, dict) and "negative_prompt" in p:
|
|
logger.warning("DMD2: ignoring negative_prompt.")
|
|
p = {k: v for k, v in p.items() if k != "negative_prompt"}
|
|
fixed.append(p)
|
|
req.prompts = fixed
|
|
|
|
def forward(self, req: OmniDiffusionRequest, **kwargs) -> DiffusionOutput:
|
|
self._sanitize_dmd2_request(req)
|
|
kwargs.pop("guidance_scale", None)
|
|
kwargs.pop("num_inference_steps", None)
|
|
return super().forward(
|
|
req,
|
|
guidance_scale=self.dmd2_guidance_scale,
|
|
num_inference_steps=self.num_inference_steps,
|
|
**kwargs,
|
|
)
|