Kandinsky6 (#8537)
Signed-off-by: leffff <levnovitskiy@gmail.com> Signed-off-by: Artem Borisov <divotionchannel@gmail.com> Co-authored-by: leffff <levnovitskiy@gmail.com> Co-authored-by: Artem Borisov <divotionchannel@gmail.com>
This commit is contained in:
38 files changed
+8574
-8
No files matched your search
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from .kandinsky6_transformer import Kandinsky6Transformer3DModel
|
||||
from .modeling_kandinsky6_audio import Kandinsky6AudioVAE
|
||||
from .modeling_kandinsky6_vae import AutoencoderKLHunyuanVideo
|
||||
from .pipeline_kandinsky6 import (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
get_kandinsky6_post_process_func,
|
||||
get_kandinsky6_pre_process_func,
|
||||
)
|
||||
from .scheduling_kandinsky6 import KandinskyFlowMatchScheduler
|
||||
|
||||
__all__ = [
|
||||
"AutoencoderKLHunyuanVideo",
|
||||
"Kandinsky6AudioVAE",
|
||||
"Kandinsky6TI2VAPipeline",
|
||||
"Kandinsky6Transformer3DModel",
|
||||
"KandinskyFlowMatchScheduler",
|
||||
"get_kandinsky6_post_process_func",
|
||||
"get_kandinsky6_pre_process_func",
|
||||
]
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Kandinsky 6 cache adapters.
|
||||
|
||||
Generic MagCache / TeaCache hooks assume a single residual stream named
|
||||
``hidden_states``. K6's fused visual blocks return ``(video, audio)``, so
|
||||
the denoise loop consults :class:`K6StepCache` instead and reuses the
|
||||
previous velocity when the schedule says the step can be skipped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Pro geometry ratios from k6_video
|
||||
# ``k6_pro_125_480_864_mCache_mOffload.yaml`` (cache.magcache.mag_ratios).
|
||||
K6_PRO_MAG_RATIOS: tuple[float, ...] = (
|
||||
1.0,
|
||||
1.0,
|
||||
0.94526286,
|
||||
0.94622554,
|
||||
1.13130749,
|
||||
1.11243134,
|
||||
1.02926022,
|
||||
1.02675114,
|
||||
1.011996,
|
||||
1.01235353,
|
||||
1.03039566,
|
||||
1.02848639,
|
||||
1.00101278,
|
||||
1.00307236,
|
||||
1.02238594,
|
||||
1.02108294,
|
||||
1.0248162,
|
||||
1.02491556,
|
||||
0.99771836,
|
||||
0.99882495,
|
||||
1.01694329,
|
||||
1.01367628,
|
||||
1.01096435,
|
||||
1.00959183,
|
||||
1.01537554,
|
||||
1.01504827,
|
||||
0.99237553,
|
||||
0.99345238,
|
||||
1.02368983,
|
||||
1.02088727,
|
||||
1.01396087,
|
||||
1.01278108,
|
||||
0.99773024,
|
||||
0.99927632,
|
||||
1.01573298,
|
||||
1.01489232,
|
||||
1.00129728,
|
||||
1.00120319,
|
||||
1.01759963,
|
||||
1.01691798,
|
||||
0.99335877,
|
||||
0.99548372,
|
||||
1.02925427,
|
||||
1.02299559,
|
||||
0.99749375,
|
||||
0.99682709,
|
||||
0.99694342,
|
||||
1.00064403,
|
||||
1.01101129,
|
||||
1.0117984,
|
||||
1.04420642,
|
||||
1.03624643,
|
||||
0.9599572,
|
||||
0.97176547,
|
||||
1.00452036,
|
||||
1.0046107,
|
||||
1.01447715,
|
||||
1.01351055,
|
||||
0.9999695,
|
||||
1.00270806,
|
||||
1.00770242,
|
||||
1.00271775,
|
||||
0.98795461,
|
||||
0.98880837,
|
||||
1.01113352,
|
||||
1.00860993,
|
||||
1.00116034,
|
||||
1.00251894,
|
||||
1.05147006,
|
||||
1.04040677,
|
||||
0.99651072,
|
||||
0.99212138,
|
||||
0.99040673,
|
||||
0.98881456,
|
||||
1.01265486,
|
||||
1.01241698,
|
||||
0.97343865,
|
||||
0.97302804,
|
||||
1.02539269,
|
||||
1.02113221,
|
||||
1.02117306,
|
||||
1.02987275,
|
||||
0.98607016,
|
||||
0.98262938,
|
||||
0.98404577,
|
||||
0.98453141,
|
||||
1.01065119,
|
||||
1.00807871,
|
||||
0.99986077,
|
||||
0.99668861,
|
||||
0.98597074,
|
||||
0.98133325,
|
||||
0.98239303,
|
||||
0.98370113,
|
||||
0.93155548,
|
||||
0.93534231,
|
||||
0.94231988,
|
||||
0.93388087,
|
||||
0.84200656,
|
||||
0.83657581,
|
||||
)
|
||||
|
||||
# Uncalibrated. Borrowed from the Qwen-Image TeaCache polynomial so the flag
|
||||
# skips steps instead of raising. Quality will not match a fitted K6 curve.
|
||||
K6_TEACACHE_COEFFICIENTS: tuple[float, ...] = (
|
||||
-4.50000000e02,
|
||||
2.80000000e02,
|
||||
-4.50000000e01,
|
||||
3.20000000e00,
|
||||
-2.00000000e-02,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class K6StepCache:
|
||||
"""Step-level skip state shared by MagCache and TeaCache."""
|
||||
|
||||
kind: str
|
||||
threshold: float
|
||||
max_skip_steps: int
|
||||
retention_ratio: float
|
||||
mag_ratios: tuple[float, ...] | None = None
|
||||
coefficients: tuple[float, ...] | None = None
|
||||
rel_l1_thresh: float = 0.2
|
||||
last_cond: object = None
|
||||
last_uncond: object = None
|
||||
accumulated_err: float = 0.0
|
||||
accumulated_ratio: float = 1.0
|
||||
accumulated_steps: int = 0
|
||||
|
||||
def should_skip(self, step_index: int) -> bool:
|
||||
if self.last_cond is None or step_index <= 0:
|
||||
return False
|
||||
if self.kind == "mag":
|
||||
ratios = self.mag_ratios or ()
|
||||
scale = float(ratios[step_index]) if step_index < len(ratios) else 1.0
|
||||
self.accumulated_ratio *= scale
|
||||
self.accumulated_steps += 1
|
||||
self.accumulated_err += abs(1.0 - self.accumulated_ratio)
|
||||
if (
|
||||
self.accumulated_err <= self.threshold
|
||||
and self.accumulated_steps <= self.max_skip_steps
|
||||
and step_index >= max(1, int(self.retention_ratio * max(step_index, 1)))
|
||||
):
|
||||
return True
|
||||
self.accumulated_ratio = 1.0
|
||||
self.accumulated_steps = 0
|
||||
self.accumulated_err = 0.0
|
||||
return False
|
||||
# TeaCache stand-in: uncalibrated polynomial of the step index.
|
||||
# The real TeaCache distance is the modulated input; this only makes
|
||||
# the flag skip steps instead of raising.
|
||||
coeff = self.coefficients or K6_TEACACHE_COEFFICIENTS
|
||||
x = float(step_index)
|
||||
predicted = sum(weight * (x ** (len(coeff) - 1 - i)) for i, weight in enumerate(coeff))
|
||||
return abs(predicted) < self.rel_l1_thresh * 1.0e4
|
||||
|
||||
def store(self, cond: object, uncond: object) -> None:
|
||||
self.last_cond = cond
|
||||
self.last_uncond = uncond
|
||||
|
||||
|
||||
def attach_mag_cache(transformer, *, threshold: float, max_skip_steps: int, retention_ratio: float) -> None:
|
||||
transformer._k6_step_cache = K6StepCache(
|
||||
kind="mag",
|
||||
threshold=threshold,
|
||||
max_skip_steps=max_skip_steps,
|
||||
retention_ratio=retention_ratio,
|
||||
mag_ratios=K6_PRO_MAG_RATIOS,
|
||||
)
|
||||
|
||||
|
||||
def attach_tea_cache(transformer, *, rel_l1_thresh: float, coefficients: tuple[float, ...] | None) -> None:
|
||||
transformer._k6_step_cache = K6StepCache(
|
||||
kind="tea",
|
||||
threshold=rel_l1_thresh if rel_l1_thresh is not None else 0.2,
|
||||
max_skip_steps=3,
|
||||
retention_ratio=0.0,
|
||||
coefficients=coefficients or K6_TEACACHE_COEFFICIENTS,
|
||||
rel_l1_thresh=rel_l1_thresh if rel_l1_thresh is not None else 0.2,
|
||||
)
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Native K6 shifted flow-matching Euler scheduler.
|
||||
|
||||
Self-contained (no ``diffusers`` dependency), matching the duck-typed
|
||||
scheduler interface (``set_timesteps`` / ``timesteps`` / ``sigmas`` /
|
||||
``step``) that other native vLLM-Omni diffusion pipelines use for their own
|
||||
per-model schedulers — see ``WanEulerScheduler`` in
|
||||
``vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler`` for the sibling
|
||||
pattern this mirrors.
|
||||
|
||||
The math reproduces ``kandinsky.core.algo.flow_matching.flow_match_timesteps``
|
||||
exactly: for ``t`` linearly spaced from 1.0 down to 0.0,
|
||||
``sigma = scale * t / (1 + (scale - 1) * t)``. This is the same shifted
|
||||
rectified-flow schedule family Wan's own scheduler uses, parameterized by
|
||||
``scheduler_scale`` (K6's name for what Wan calls ``shift``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class KandinskyFlowMatchSchedulerOutput:
|
||||
prev_sample: torch.Tensor
|
||||
|
||||
|
||||
class KandinskyFlowMatchScheduler:
|
||||
"""Stateful Euler stepper over K6's shifted flow-matching sigma schedule.
|
||||
|
||||
``timesteps`` holds the model-scale (``[0, 1000]``-ish) values fed to the
|
||||
DiT's ``time`` input; ``sigmas`` holds the same schedule unscaled
|
||||
(``num_inference_steps + 1`` entries, running 1.0 -> 0.0), needed to
|
||||
derive a step size without consuming a stateful ``step()`` call when one
|
||||
modality is stepped implicitly alongside another (see the ``vllm`` port's
|
||||
``denoise_loop`` override).
|
||||
"""
|
||||
|
||||
order = 1
|
||||
|
||||
def __init__(self, scheduler_scale: float = 1.0, device: torch.device | str = "cpu") -> None:
|
||||
self.scheduler_scale = float(scheduler_scale)
|
||||
self.device = device
|
||||
self.config = SimpleNamespace(scheduler_scale=self.scheduler_scale)
|
||||
|
||||
self._step_index: int | None = None
|
||||
self.timesteps = torch.empty(0, dtype=torch.float32)
|
||||
self.sigmas = torch.empty(0, dtype=torch.float32)
|
||||
|
||||
self.set_timesteps(num_inference_steps=1, device=self.device)
|
||||
|
||||
@property
|
||||
def step_index(self) -> int | None:
|
||||
return self._step_index
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int,
|
||||
device: torch.device | str | None = None,
|
||||
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
|
||||
) -> None:
|
||||
device = device or self.device
|
||||
t = torch.linspace(1.0, 0.0, int(num_inference_steps) + 1, device=device, dtype=torch.float32)
|
||||
scale = self.scheduler_scale
|
||||
self.sigmas = scale * t / (1 + (scale - 1) * t)
|
||||
# DiT `time` input is model-scale, i.e. sigma * 1000 (matches the core
|
||||
# algo loop's `t.unsqueeze(0).expand(bs) * 1000`).
|
||||
self.timesteps = self.sigmas[:-1] * 1000.0
|
||||
self._step_index = None
|
||||
|
||||
def index_for_timestep(self, timestep: torch.Tensor) -> int:
|
||||
timestep_t = torch.as_tensor(timestep, device=self.timesteps.device, dtype=self.timesteps.dtype)
|
||||
return int(torch.argmin(torch.abs(self.timesteps - timestep_t)).item())
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
sample: torch.Tensor,
|
||||
return_dict: bool = True,
|
||||
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
|
||||
) -> KandinskyFlowMatchSchedulerOutput | tuple[torch.Tensor]:
|
||||
if self._step_index is None:
|
||||
self._step_index = self.index_for_timestep(timestep)
|
||||
|
||||
sigma = self.sigmas[self._step_index]
|
||||
sigma_next = self.sigmas[self._step_index + 1]
|
||||
dt = (sigma_next - sigma).to(device=sample.device, dtype=sample.dtype)
|
||||
prev_sample = sample + dt * model_output
|
||||
|
||||
self._step_index += 1
|
||||
|
||||
if not return_dict:
|
||||
return (prev_sample,)
|
||||
return KandinskyFlowMatchSchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return int(self.timesteps.shape[0])
|
||||
Reference in new issue
Block a user