[Model] SupportHunyuanImage3 Diffusion Model in vllm-omni (#1085)
Signed-off-by: Semmer2 <semmer@live.cn>
This commit is contained in:
1 parent
26ba1e4de8
commit
5fea4823b7
13 files changed
+5695
-23
No files matched your search
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Hunyuan Image 3 diffusion model components."""
|
||||
|
||||
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_image_3_transformer import (
|
||||
HunyuanImage3Model,
|
||||
HunyuanImage3Text2ImagePipeline,
|
||||
)
|
||||
from vllm_omni.diffusion.models.hunyuan_image_3.pipeline_hunyuan_image_3 import (
|
||||
HunyuanImage3Pipeline,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"HunyuanImage3Pipeline",
|
||||
"HunyuanImage3Model",
|
||||
"HunyuanImage3Text2ImagePipeline",
|
||||
]
|
||||
@@ -0,0 +1,668 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
from diffusers.models.modeling_utils import ModelMixin
|
||||
from diffusers.utils import BaseOutput
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from einops import rearrange
|
||||
from torch import Tensor, nn
|
||||
|
||||
|
||||
class DiagonalGaussianDistribution:
|
||||
def __init__(self, parameters: torch.Tensor, deterministic: bool = False):
|
||||
if parameters.ndim == 3:
|
||||
dim = 2 # (B, L, C)
|
||||
elif parameters.ndim == 5 or parameters.ndim == 4:
|
||||
dim = 1 # (B, C, T, H ,W) / (B, C, H, W)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
self.parameters = parameters
|
||||
self.mean, self.logvar = torch.chunk(parameters, 2, dim=dim)
|
||||
self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
|
||||
self.deterministic = deterministic
|
||||
self.std = torch.exp(0.5 * self.logvar)
|
||||
self.var = torch.exp(self.logvar)
|
||||
if self.deterministic:
|
||||
self.var = self.std = torch.zeros_like(
|
||||
self.mean, device=self.parameters.device, dtype=self.parameters.dtype
|
||||
)
|
||||
|
||||
def sample(self, generator: torch.Generator | None = None) -> torch.FloatTensor:
|
||||
# make sure sample is on the same device as the parameters and has same dtype
|
||||
sample = randn_tensor(
|
||||
self.mean.shape,
|
||||
generator=generator,
|
||||
device=self.parameters.device,
|
||||
dtype=self.parameters.dtype,
|
||||
)
|
||||
x = self.mean + self.std * sample
|
||||
return x
|
||||
|
||||
|
||||
@dataclass
|
||||
class DecoderOutput(BaseOutput):
|
||||
sample: torch.FloatTensor
|
||||
posterior: DiagonalGaussianDistribution | None = None
|
||||
|
||||
|
||||
def swish(x: Tensor) -> Tensor:
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
def forward_with_checkpointing(module, *inputs, use_checkpointing=False):
|
||||
def create_custom_forward(module):
|
||||
def custom_forward(*inputs):
|
||||
return module(*inputs)
|
||||
|
||||
return custom_forward
|
||||
|
||||
if use_checkpointing:
|
||||
return torch.utils.checkpoint.checkpoint(create_custom_forward(module), *inputs, use_reentrant=False)
|
||||
else:
|
||||
return module(*inputs)
|
||||
|
||||
|
||||
class Conv3d(nn.Conv3d):
|
||||
"""
|
||||
Perform Conv3d on patches with numerical differences from nn.Conv3d within 1e-5.
|
||||
Only symmetric padding is supported.
|
||||
"""
|
||||
|
||||
def forward(self, input):
|
||||
B, C, T, H, W = input.shape
|
||||
memory_count = (C * T * H * W) * 2 / 1024**3
|
||||
if memory_count > 2:
|
||||
n_split = math.ceil(memory_count / 2)
|
||||
assert n_split >= 2
|
||||
chunks = torch.chunk(input, chunks=n_split, dim=-3)
|
||||
padded_chunks = []
|
||||
for i in range(len(chunks)):
|
||||
if self.padding[0] > 0:
|
||||
padded_chunk = F.pad(
|
||||
chunks[i],
|
||||
(0, 0, 0, 0, self.padding[0], self.padding[0]),
|
||||
mode="constant" if self.padding_mode == "zeros" else self.padding_mode,
|
||||
value=0,
|
||||
)
|
||||
if i > 0:
|
||||
padded_chunk[:, :, : self.padding[0]] = chunks[i - 1][:, :, -self.padding[0] :]
|
||||
if i < len(chunks) - 1:
|
||||
padded_chunk[:, :, -self.padding[0] :] = chunks[i + 1][:, :, : self.padding[0]]
|
||||
else:
|
||||
padded_chunk = chunks[i]
|
||||
padded_chunks.append(padded_chunk)
|
||||
padding_bak = self.padding
|
||||
self.padding = (0, self.padding[1], self.padding[2])
|
||||
outputs = []
|
||||
for i in range(len(padded_chunks)):
|
||||
outputs.append(super().forward(padded_chunks[i]))
|
||||
self.padding = padding_bak
|
||||
return torch.cat(outputs, dim=-3)
|
||||
else:
|
||||
return super().forward(input)
|
||||
|
||||
|
||||
class AttnBlock(nn.Module):
|
||||
"""Attention with torch sdpa implementation."""
|
||||
|
||||
def __init__(self, in_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
|
||||
self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
|
||||
|
||||
self.q = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.k = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.v = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1)
|
||||
|
||||
def attention(self, h_: Tensor) -> Tensor:
|
||||
h_ = self.norm(h_)
|
||||
q = self.q(h_)
|
||||
k = self.k(h_)
|
||||
v = self.v(h_)
|
||||
|
||||
b, c, f, h, w = q.shape
|
||||
q = rearrange(q, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
k = rearrange(k, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
v = rearrange(v, "b c f h w -> b 1 (f h w) c").contiguous()
|
||||
h_ = nn.functional.scaled_dot_product_attention(q, k, v)
|
||||
|
||||
return rearrange(h_, "b 1 (f h w) c -> b c f h w", f=f, h=h, w=w, c=c, b=b)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return x + self.proj_out(self.attention(x))
|
||||
|
||||
|
||||
class ResnetBlock(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
out_channels = in_channels if out_channels is None else out_channels
|
||||
self.out_channels = out_channels
|
||||
|
||||
self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
|
||||
self.conv1 = Conv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True)
|
||||
self.conv2 = Conv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
if self.in_channels != self.out_channels:
|
||||
self.nin_shortcut = Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
|
||||
|
||||
def forward(self, x):
|
||||
h = x
|
||||
h = self.norm1(h)
|
||||
h = swish(h)
|
||||
h = self.conv1(h)
|
||||
|
||||
h = self.norm2(h)
|
||||
h = swish(h)
|
||||
h = self.conv2(h)
|
||||
|
||||
if self.in_channels != self.out_channels:
|
||||
x = self.nin_shortcut(x)
|
||||
return x + h
|
||||
|
||||
|
||||
class DownsampleDCAE(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True):
|
||||
super().__init__()
|
||||
factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2
|
||||
assert out_channels % factor == 0
|
||||
self.conv = Conv3d(in_channels, out_channels // factor, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.add_temporal_downsample = add_temporal_downsample
|
||||
self.group_size = factor * in_channels // out_channels
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
r1 = 2 if self.add_temporal_downsample else 1
|
||||
h = self.conv(x)
|
||||
h = rearrange(h, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)
|
||||
shortcut = rearrange(x, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)
|
||||
|
||||
B, C, T, H, W = shortcut.shape
|
||||
shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean(dim=2)
|
||||
return h + shortcut
|
||||
|
||||
|
||||
class UpsampleDCAE(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True):
|
||||
super().__init__()
|
||||
factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2
|
||||
self.conv = Conv3d(in_channels, out_channels * factor, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.add_temporal_upsample = add_temporal_upsample
|
||||
self.repeats = factor * out_channels // in_channels
|
||||
|
||||
def forward(self, x: Tensor):
|
||||
r1 = 2 if self.add_temporal_upsample else 1
|
||||
h = self.conv(x)
|
||||
h = rearrange(h, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)
|
||||
shortcut = x.repeat_interleave(repeats=self.repeats, dim=1)
|
||||
shortcut = rearrange(shortcut, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)
|
||||
return h + shortcut
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
"""
|
||||
The encoder network of AutoencoderKLConv3D.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
z_channels: int,
|
||||
block_out_channels: tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
downsample_match_channel: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
assert block_out_channels[-1] % (2 * z_channels) == 0
|
||||
|
||||
self.z_channels = z_channels
|
||||
self.block_out_channels = block_out_channels
|
||||
self.num_res_blocks = num_res_blocks
|
||||
|
||||
# downsampling
|
||||
self.conv_in = Conv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.down = nn.ModuleList()
|
||||
block_in = block_out_channels[0]
|
||||
for i_level, ch in enumerate(block_out_channels):
|
||||
block = nn.ModuleList()
|
||||
block_out = ch
|
||||
for _ in range(self.num_res_blocks):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
|
||||
add_spatial_downsample = bool(i_level < np.log2(ffactor_spatial))
|
||||
add_temporal_downsample = add_spatial_downsample and bool(
|
||||
i_level >= np.log2(ffactor_spatial // ffactor_temporal)
|
||||
)
|
||||
if add_spatial_downsample or add_temporal_downsample:
|
||||
assert i_level < len(block_out_channels) - 1
|
||||
block_out = block_out_channels[i_level + 1] if downsample_match_channel else block_in
|
||||
down.downsample = DownsampleDCAE(block_in, block_out, add_temporal_downsample)
|
||||
block_in = block_out
|
||||
self.down.append(down)
|
||||
|
||||
# middle
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
# end
|
||||
self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
|
||||
self.conv_out = Conv3d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
use_checkpointing = bool(self.training and self.gradient_checkpointing)
|
||||
|
||||
# downsampling
|
||||
h = self.conv_in(x)
|
||||
for i_level in range(len(self.block_out_channels)):
|
||||
for i_block in range(self.num_res_blocks):
|
||||
h = forward_with_checkpointing(
|
||||
self.down[i_level].block[i_block], h, use_checkpointing=use_checkpointing
|
||||
)
|
||||
if hasattr(self.down[i_level], "downsample"):
|
||||
h = forward_with_checkpointing(self.down[i_level].downsample, h, use_checkpointing=use_checkpointing)
|
||||
|
||||
# middle
|
||||
h = forward_with_checkpointing(self.mid.block_1, h, use_checkpointing=use_checkpointing)
|
||||
h = forward_with_checkpointing(self.mid.attn_1, h, use_checkpointing=use_checkpointing)
|
||||
h = forward_with_checkpointing(self.mid.block_2, h, use_checkpointing=use_checkpointing)
|
||||
|
||||
# end
|
||||
group_size = self.block_out_channels[-1] // (2 * self.z_channels)
|
||||
shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean(dim=2)
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
h += shortcut
|
||||
return h
|
||||
|
||||
|
||||
class Decoder(nn.Module):
|
||||
"""
|
||||
The decoder network of AutoencoderKLConv3D.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
z_channels: int,
|
||||
out_channels: int,
|
||||
block_out_channels: tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
upsample_match_channel: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
assert block_out_channels[0] % z_channels == 0
|
||||
|
||||
self.z_channels = z_channels
|
||||
self.block_out_channels = block_out_channels
|
||||
self.num_res_blocks = num_res_blocks
|
||||
|
||||
# z to block_in
|
||||
block_in = block_out_channels[0]
|
||||
self.conv_in = Conv3d(z_channels, block_in, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
# middle
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
self.mid.attn_1 = AttnBlock(block_in)
|
||||
self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
|
||||
|
||||
# upsampling
|
||||
self.up = nn.ModuleList()
|
||||
for i_level, ch in enumerate(block_out_channels):
|
||||
block = nn.ModuleList()
|
||||
block_out = ch
|
||||
for _ in range(self.num_res_blocks + 1):
|
||||
block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
|
||||
block_in = block_out
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
|
||||
add_spatial_upsample = bool(i_level < np.log2(ffactor_spatial))
|
||||
add_temporal_upsample = bool(i_level < np.log2(ffactor_temporal))
|
||||
if add_spatial_upsample or add_temporal_upsample:
|
||||
assert i_level < len(block_out_channels) - 1
|
||||
block_out = block_out_channels[i_level + 1] if upsample_match_channel else block_in
|
||||
up.upsample = UpsampleDCAE(block_in, block_out, add_temporal_upsample)
|
||||
block_in = block_out
|
||||
self.up.append(up)
|
||||
|
||||
# end
|
||||
self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
|
||||
self.conv_out = Conv3d(block_in, out_channels, kernel_size=3, stride=1, padding=1)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
def forward(self, z: Tensor) -> Tensor:
|
||||
use_checkpointing = bool(self.training and self.gradient_checkpointing)
|
||||
|
||||
# z to block_in
|
||||
repeats = self.block_out_channels[0] // (self.z_channels)
|
||||
h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1)
|
||||
|
||||
# middle
|
||||
h = forward_with_checkpointing(self.mid.block_1, h, use_checkpointing=use_checkpointing)
|
||||
h = forward_with_checkpointing(self.mid.attn_1, h, use_checkpointing=use_checkpointing)
|
||||
h = forward_with_checkpointing(self.mid.block_2, h, use_checkpointing=use_checkpointing)
|
||||
|
||||
# upsampling
|
||||
for i_level in range(len(self.block_out_channels)):
|
||||
for i_block in range(self.num_res_blocks + 1):
|
||||
h = forward_with_checkpointing(self.up[i_level].block[i_block], h, use_checkpointing=use_checkpointing)
|
||||
if hasattr(self.up[i_level], "upsample"):
|
||||
h = forward_with_checkpointing(self.up[i_level].upsample, h, use_checkpointing=use_checkpointing)
|
||||
|
||||
# end
|
||||
h = self.norm_out(h)
|
||||
h = swish(h)
|
||||
h = self.conv_out(h)
|
||||
return h
|
||||
|
||||
|
||||
class AutoencoderKLConv3D(ModelMixin, ConfigMixin):
|
||||
"""
|
||||
Autoencoder model with KL-regularized latent space based on 3D convolutions.
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
out_channels: int,
|
||||
latent_channels: int,
|
||||
block_out_channels: tuple[int, ...],
|
||||
layers_per_block: int,
|
||||
ffactor_spatial: int,
|
||||
ffactor_temporal: int,
|
||||
sample_size: int,
|
||||
sample_tsize: int,
|
||||
scaling_factor: float | None = None,
|
||||
shift_factor: float | None = None,
|
||||
downsample_match_channel: bool = True,
|
||||
upsample_match_channel: bool = True,
|
||||
only_encoder: bool = False, # only build encoder for saving memory
|
||||
only_decoder: bool = False, # only build decoder for saving memory
|
||||
):
|
||||
super().__init__()
|
||||
self.ffactor_spatial = ffactor_spatial
|
||||
self.ffactor_temporal = ffactor_temporal
|
||||
self.scaling_factor = scaling_factor
|
||||
self.shift_factor = shift_factor
|
||||
|
||||
# build model
|
||||
if not only_decoder:
|
||||
self.encoder = Encoder(
|
||||
in_channels=in_channels,
|
||||
z_channels=latent_channels,
|
||||
block_out_channels=block_out_channels,
|
||||
num_res_blocks=layers_per_block,
|
||||
ffactor_spatial=ffactor_spatial,
|
||||
ffactor_temporal=ffactor_temporal,
|
||||
downsample_match_channel=downsample_match_channel,
|
||||
)
|
||||
if not only_encoder:
|
||||
self.decoder = Decoder(
|
||||
z_channels=latent_channels,
|
||||
out_channels=out_channels,
|
||||
block_out_channels=list(reversed(block_out_channels)),
|
||||
num_res_blocks=layers_per_block,
|
||||
ffactor_spatial=ffactor_spatial,
|
||||
ffactor_temporal=ffactor_temporal,
|
||||
upsample_match_channel=upsample_match_channel,
|
||||
)
|
||||
|
||||
# slicing and tiling related
|
||||
self.use_slicing = False
|
||||
self.slicing_bsz = 1
|
||||
self.use_spatial_tiling = False
|
||||
self.use_temporal_tiling = False
|
||||
self.use_tiling_during_training = False
|
||||
|
||||
# only relevant if vae tiling is enabled
|
||||
self.tile_sample_min_size = sample_size
|
||||
self.tile_latent_min_size = sample_size // ffactor_spatial
|
||||
self.tile_sample_min_tsize = sample_tsize
|
||||
self.tile_latent_min_tsize = sample_tsize // ffactor_temporal
|
||||
self.tile_overlap_factor = 0.25
|
||||
|
||||
# use torch.compile for faster encode speed
|
||||
self.use_compile = False
|
||||
|
||||
def _set_gradient_checkpointing(self, module, value=False):
|
||||
if isinstance(module, (Encoder, Decoder)):
|
||||
module.gradient_checkpointing = value
|
||||
|
||||
def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (
|
||||
x / blend_extent
|
||||
)
|
||||
return b
|
||||
|
||||
def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-2], b.shape[-2], blend_extent)
|
||||
for y in range(blend_extent):
|
||||
b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (
|
||||
y / blend_extent
|
||||
)
|
||||
return b
|
||||
|
||||
def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):
|
||||
blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (1 - x / blend_extent) + b[:, :, x, :, :] * (
|
||||
x / blend_extent
|
||||
)
|
||||
return b
|
||||
|
||||
def spatial_tiled_encode(self, x: torch.Tensor):
|
||||
"""spatial tailing for frames"""
|
||||
B, C, T, H, W = x.shape
|
||||
overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) # 256 * (1 - 0.25) = 192
|
||||
blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) # 8 * 0.25 = 2
|
||||
row_limit = self.tile_latent_min_size - blend_extent # 8 - 2 = 6
|
||||
|
||||
rows = []
|
||||
for i in range(0, H, overlap_size):
|
||||
row = []
|
||||
for j in range(0, W, overlap_size):
|
||||
tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size]
|
||||
tile = self.encoder(tile)
|
||||
row.append(tile)
|
||||
rows.append(row)
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :, :row_limit, :row_limit])
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
moments = torch.cat(result_rows, dim=-2)
|
||||
return moments
|
||||
|
||||
def temporal_tiled_encode(self, x: torch.Tensor):
|
||||
"""temporal tailing for frames"""
|
||||
B, C, T, H, W = x.shape
|
||||
overlap_size = int(self.tile_sample_min_tsize * (1 - self.tile_overlap_factor)) # 64 * (1 - 0.25) = 48
|
||||
blend_extent = int(self.tile_latent_min_tsize * self.tile_overlap_factor) # 8 * 0.25 = 2
|
||||
t_limit = self.tile_latent_min_tsize - blend_extent # 8 - 2 = 6
|
||||
|
||||
row = []
|
||||
for i in range(0, T, overlap_size):
|
||||
tile = x[:, :, i : i + self.tile_sample_min_tsize, :, :]
|
||||
if self.use_spatial_tiling and (
|
||||
tile.shape[-1] > self.tile_sample_min_size or tile.shape[-2] > self.tile_sample_min_size
|
||||
):
|
||||
tile = self.spatial_tiled_encode(tile)
|
||||
else:
|
||||
tile = self.encoder(tile)
|
||||
row.append(tile)
|
||||
result_row = []
|
||||
for i, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_t(row[i - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :t_limit, :, :])
|
||||
moments = torch.cat(result_row, dim=-3)
|
||||
return moments
|
||||
|
||||
def spatial_tiled_decode(self, z: torch.Tensor):
|
||||
"""spatial tailing for frames"""
|
||||
B, C, T, H, W = z.shape
|
||||
overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) # 8 * (1 - 0.25) = 6
|
||||
blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) # 256 * 0.25 = 64
|
||||
row_limit = self.tile_sample_min_size - blend_extent # 256 - 64 = 192
|
||||
|
||||
rows = []
|
||||
for i in range(0, H, overlap_size):
|
||||
row = []
|
||||
for j in range(0, W, overlap_size):
|
||||
tile = z[:, :, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size]
|
||||
decoded = self.decoder(tile)
|
||||
row.append(decoded)
|
||||
rows.append(row)
|
||||
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_extent)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :, :row_limit, :row_limit])
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
dec = torch.cat(result_rows, dim=-2)
|
||||
return dec
|
||||
|
||||
def temporal_tiled_decode(self, z: torch.Tensor):
|
||||
"""temporal tailing for frames"""
|
||||
B, C, T, H, W = z.shape
|
||||
overlap_size = int(self.tile_latent_min_tsize * (1 - self.tile_overlap_factor)) # 8 * (1 - 0.25) = 6
|
||||
blend_extent = int(self.tile_sample_min_tsize * self.tile_overlap_factor) # 64 * 0.25 = 16
|
||||
t_limit = self.tile_sample_min_tsize - blend_extent # 64 - 16 = 48
|
||||
assert 0 < overlap_size < self.tile_latent_min_tsize
|
||||
|
||||
row = []
|
||||
for i in range(0, T, overlap_size):
|
||||
tile = z[:, :, i : i + self.tile_latent_min_tsize, :, :]
|
||||
if self.use_spatial_tiling and (
|
||||
tile.shape[-1] > self.tile_latent_min_size or tile.shape[-2] > self.tile_latent_min_size
|
||||
):
|
||||
decoded = self.spatial_tiled_decode(tile)
|
||||
else:
|
||||
decoded = self.decoder(tile)
|
||||
row.append(decoded)
|
||||
|
||||
result_row = []
|
||||
for i, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_t(row[i - 1], tile, blend_extent)
|
||||
result_row.append(tile[:, :, :t_limit, :, :])
|
||||
dec = torch.cat(result_row, dim=-3)
|
||||
return dec
|
||||
|
||||
def encode(self, x: Tensor, return_dict: bool = True):
|
||||
"""
|
||||
Encodes the input by passing through the encoder network.
|
||||
Support slicing and tiling for memory efficiency.
|
||||
"""
|
||||
|
||||
def _encode(x):
|
||||
if self.use_temporal_tiling and x.shape[-3] > self.tile_sample_min_tsize:
|
||||
return self.temporal_tiled_encode(x)
|
||||
if self.use_spatial_tiling and (
|
||||
x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size
|
||||
):
|
||||
return self.spatial_tiled_encode(x)
|
||||
|
||||
if self.use_compile:
|
||||
|
||||
@torch.compile
|
||||
def encoder(x):
|
||||
return self.encoder(x)
|
||||
|
||||
return encoder(x)
|
||||
return self.encoder(x)
|
||||
|
||||
if len(x.shape) != 5: # (B, C, T, H, W)
|
||||
x = x[:, :, None]
|
||||
assert len(x.shape) == 5 # (B, C, T, H, W)
|
||||
if x.shape[2] == 1:
|
||||
x = x.expand(-1, -1, self.ffactor_temporal, -1, -1)
|
||||
else:
|
||||
assert x.shape[2] != self.ffactor_temporal and x.shape[2] % self.ffactor_temporal == 0
|
||||
|
||||
if self.use_slicing and x.shape[0] > 1:
|
||||
if self.slicing_bsz == 1:
|
||||
encoded_slices = [_encode(x_slice) for x_slice in x.split(1)]
|
||||
else:
|
||||
sections = [self.slicing_bsz] * (x.shape[0] // self.slicing_bsz)
|
||||
if x.shape[0] % self.slicing_bsz != 0:
|
||||
sections.append(x.shape[0] % self.slicing_bsz)
|
||||
encoded_slices = [_encode(x_slice) for x_slice in x.split(sections)]
|
||||
h = torch.cat(encoded_slices)
|
||||
else:
|
||||
h = _encode(x)
|
||||
posterior = DiagonalGaussianDistribution(h)
|
||||
|
||||
if not return_dict:
|
||||
return (posterior,)
|
||||
|
||||
return AutoencoderKLOutput(latent_dist=posterior)
|
||||
|
||||
def decode(self, z: Tensor, return_dict: bool = True, generator=None):
|
||||
"""
|
||||
Decodes the input by passing through the decoder network.
|
||||
Support slicing and tiling for memory efficiency.
|
||||
"""
|
||||
|
||||
def _decode(z):
|
||||
if self.use_temporal_tiling and z.shape[-3] > self.tile_latent_min_tsize:
|
||||
return self.temporal_tiled_decode(z)
|
||||
if self.use_spatial_tiling and (
|
||||
z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size
|
||||
):
|
||||
return self.spatial_tiled_decode(z)
|
||||
return self.decoder(z)
|
||||
|
||||
if self.use_slicing and z.shape[0] > 1:
|
||||
decoded_slices = [_decode(z_slice) for z_slice in z.split(1)]
|
||||
decoded = torch.cat(decoded_slices)
|
||||
else:
|
||||
decoded = _decode(z)
|
||||
|
||||
if z.shape[-3] == 1:
|
||||
decoded = decoded[:, :, -1:]
|
||||
|
||||
if not return_dict:
|
||||
return (decoded,)
|
||||
|
||||
return DecoderOutput(sample=decoded)
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user