Kandinsky6 (#8537)

Signed-off-by: leffff <levnovitskiy@gmail.com>
Signed-off-by: Artem Borisov <divotionchannel@gmail.com>
Co-authored-by: leffff <levnovitskiy@gmail.com>
Co-authored-by: Artem Borisov <divotionchannel@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-10-06 15:55:15 +08:00
1 parent 1f63e97a40
commit 5f95115e70
38 files changed
+8574 -8

No files matched your search

@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from .kandinsky6_transformer import Kandinsky6Transformer3DModel
from .modeling_kandinsky6_audio import Kandinsky6AudioVAE
from .modeling_kandinsky6_vae import AutoencoderKLHunyuanVideo
from .pipeline_kandinsky6 import (
Kandinsky6TI2VAPipeline,
get_kandinsky6_post_process_func,
get_kandinsky6_pre_process_func,
)
from .scheduling_kandinsky6 import KandinskyFlowMatchScheduler
__all__ = [
"AutoencoderKLHunyuanVideo",
"Kandinsky6AudioVAE",
"Kandinsky6TI2VAPipeline",
"Kandinsky6Transformer3DModel",
"KandinskyFlowMatchScheduler",
"get_kandinsky6_post_process_func",
"get_kandinsky6_pre_process_func",
]
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Kandinsky 6 cache adapters.
Generic MagCache / TeaCache hooks assume a single residual stream named
``hidden_states``. K6's fused visual blocks return ``(video, audio)``, so
the denoise loop consults :class:`K6StepCache` instead and reuses the
previous velocity when the schedule says the step can be skipped.
"""
from __future__ import annotations
from dataclasses import dataclass
# Pro geometry ratios from k6_video
# ``k6_pro_125_480_864_mCache_mOffload.yaml`` (cache.magcache.mag_ratios).
K6_PRO_MAG_RATIOS: tuple[float, ...] = (
1.0,
1.0,
0.94526286,
0.94622554,
1.13130749,
1.11243134,
1.02926022,
1.02675114,
1.011996,
1.01235353,
1.03039566,
1.02848639,
1.00101278,
1.00307236,
1.02238594,
1.02108294,
1.0248162,
1.02491556,
0.99771836,
0.99882495,
1.01694329,
1.01367628,
1.01096435,
1.00959183,
1.01537554,
1.01504827,
0.99237553,
0.99345238,
1.02368983,
1.02088727,
1.01396087,
1.01278108,
0.99773024,
0.99927632,
1.01573298,
1.01489232,
1.00129728,
1.00120319,
1.01759963,
1.01691798,
0.99335877,
0.99548372,
1.02925427,
1.02299559,
0.99749375,
0.99682709,
0.99694342,
1.00064403,
1.01101129,
1.0117984,
1.04420642,
1.03624643,
0.9599572,
0.97176547,
1.00452036,
1.0046107,
1.01447715,
1.01351055,
0.9999695,
1.00270806,
1.00770242,
1.00271775,
0.98795461,
0.98880837,
1.01113352,
1.00860993,
1.00116034,
1.00251894,
1.05147006,
1.04040677,
0.99651072,
0.99212138,
0.99040673,
0.98881456,
1.01265486,
1.01241698,
0.97343865,
0.97302804,
1.02539269,
1.02113221,
1.02117306,
1.02987275,
0.98607016,
0.98262938,
0.98404577,
0.98453141,
1.01065119,
1.00807871,
0.99986077,
0.99668861,
0.98597074,
0.98133325,
0.98239303,
0.98370113,
0.93155548,
0.93534231,
0.94231988,
0.93388087,
0.84200656,
0.83657581,
)
# Uncalibrated. Borrowed from the Qwen-Image TeaCache polynomial so the flag
# skips steps instead of raising. Quality will not match a fitted K6 curve.
K6_TEACACHE_COEFFICIENTS: tuple[float, ...] = (
-4.50000000e02,
2.80000000e02,
-4.50000000e01,
3.20000000e00,
-2.00000000e-02,
)
@dataclass
class K6StepCache:
"""Step-level skip state shared by MagCache and TeaCache."""
kind: str
threshold: float
max_skip_steps: int
retention_ratio: float
mag_ratios: tuple[float, ...] | None = None
coefficients: tuple[float, ...] | None = None
rel_l1_thresh: float = 0.2
last_cond: object = None
last_uncond: object = None
accumulated_err: float = 0.0
accumulated_ratio: float = 1.0
accumulated_steps: int = 0
def should_skip(self, step_index: int) -> bool:
if self.last_cond is None or step_index <= 0:
return False
if self.kind == "mag":
ratios = self.mag_ratios or ()
scale = float(ratios[step_index]) if step_index < len(ratios) else 1.0
self.accumulated_ratio *= scale
self.accumulated_steps += 1
self.accumulated_err += abs(1.0 - self.accumulated_ratio)
if (
self.accumulated_err <= self.threshold
and self.accumulated_steps <= self.max_skip_steps
and step_index >= max(1, int(self.retention_ratio * max(step_index, 1)))
):
return True
self.accumulated_ratio = 1.0
self.accumulated_steps = 0
self.accumulated_err = 0.0
return False
# TeaCache stand-in: uncalibrated polynomial of the step index.
# The real TeaCache distance is the modulated input; this only makes
# the flag skip steps instead of raising.
coeff = self.coefficients or K6_TEACACHE_COEFFICIENTS
x = float(step_index)
predicted = sum(weight * (x ** (len(coeff) - 1 - i)) for i, weight in enumerate(coeff))
return abs(predicted) < self.rel_l1_thresh * 1.0e4
def store(self, cond: object, uncond: object) -> None:
self.last_cond = cond
self.last_uncond = uncond
def attach_mag_cache(transformer, *, threshold: float, max_skip_steps: int, retention_ratio: float) -> None:
transformer._k6_step_cache = K6StepCache(
kind="mag",
threshold=threshold,
max_skip_steps=max_skip_steps,
retention_ratio=retention_ratio,
mag_ratios=K6_PRO_MAG_RATIOS,
)
def attach_tea_cache(transformer, *, rel_l1_thresh: float, coefficients: tuple[float, ...] | None) -> None:
transformer._k6_step_cache = K6StepCache(
kind="tea",
threshold=rel_l1_thresh if rel_l1_thresh is not None else 0.2,
max_skip_steps=3,
retention_ratio=0.0,
coefficients=coefficients or K6_TEACACHE_COEFFICIENTS,
rel_l1_thresh=rel_l1_thresh if rel_l1_thresh is not None else 0.2,
)
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Native K6 shifted flow-matching Euler scheduler.
Self-contained (no ``diffusers`` dependency), matching the duck-typed
scheduler interface (``set_timesteps`` / ``timesteps`` / ``sigmas`` /
``step``) that other native vLLM-Omni diffusion pipelines use for their own
per-model schedulers — see ``WanEulerScheduler`` in
``vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler`` for the sibling
pattern this mirrors.
The math reproduces ``kandinsky.core.algo.flow_matching.flow_match_timesteps``
exactly: for ``t`` linearly spaced from 1.0 down to 0.0,
``sigma = scale * t / (1 + (scale - 1) * t)``. This is the same shifted
rectified-flow schedule family Wan's own scheduler uses, parameterized by
``scheduler_scale`` (K6's name for what Wan calls ``shift``).
"""
from __future__ import annotations
from dataclasses import dataclass
from types import SimpleNamespace
import torch
@dataclass
class KandinskyFlowMatchSchedulerOutput:
prev_sample: torch.Tensor
class KandinskyFlowMatchScheduler:
"""Stateful Euler stepper over K6's shifted flow-matching sigma schedule.
``timesteps`` holds the model-scale (``[0, 1000]``-ish) values fed to the
DiT's ``time`` input; ``sigmas`` holds the same schedule unscaled
(``num_inference_steps + 1`` entries, running 1.0 -> 0.0), needed to
derive a step size without consuming a stateful ``step()`` call when one
modality is stepped implicitly alongside another (see the ``vllm`` port's
``denoise_loop`` override).
"""
order = 1
def __init__(self, scheduler_scale: float = 1.0, device: torch.device | str = "cpu") -> None:
self.scheduler_scale = float(scheduler_scale)
self.device = device
self.config = SimpleNamespace(scheduler_scale=self.scheduler_scale)
self._step_index: int | None = None
self.timesteps = torch.empty(0, dtype=torch.float32)
self.sigmas = torch.empty(0, dtype=torch.float32)
self.set_timesteps(num_inference_steps=1, device=self.device)
@property
def step_index(self) -> int | None:
return self._step_index
def set_timesteps(
self,
num_inference_steps: int,
device: torch.device | str | None = None,
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
) -> None:
device = device or self.device
t = torch.linspace(1.0, 0.0, int(num_inference_steps) + 1, device=device, dtype=torch.float32)
scale = self.scheduler_scale
self.sigmas = scale * t / (1 + (scale - 1) * t)
# DiT `time` input is model-scale, i.e. sigma * 1000 (matches the core
# algo loop's `t.unsqueeze(0).expand(bs) * 1000`).
self.timesteps = self.sigmas[:-1] * 1000.0
self._step_index = None
def index_for_timestep(self, timestep: torch.Tensor) -> int:
timestep_t = torch.as_tensor(timestep, device=self.timesteps.device, dtype=self.timesteps.dtype)
return int(torch.argmin(torch.abs(self.timesteps - timestep_t)).item())
def step(
self,
model_output: torch.Tensor,
timestep: torch.Tensor,
sample: torch.Tensor,
return_dict: bool = True,
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
) -> KandinskyFlowMatchSchedulerOutput | tuple[torch.Tensor]:
if self._step_index is None:
self._step_index = self.index_for_timestep(timestep)
sigma = self.sigmas[self._step_index]
sigma_next = self.sigmas[self._step_index + 1]
dt = (sigma_next - sigma).to(device=sample.device, dtype=sample.dtype)
prev_sample = sample + dt * model_output
self._step_index += 1
if not return_dict:
return (prev_sample,)
return KandinskyFlowMatchSchedulerOutput(prev_sample=prev_sample)
def __len__(self) -> int:
return int(self.timesteps.shape[0])