[Perf][Diffusion] Add LTX2 channels-last VAE path
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
(cherry picked from commit dd6f32cb47)
This commit is contained in:
1 parent
fcf1085395
commit
81e5e8e3bb
12 files changed
+250
No files matched your search
@@ -779,6 +779,30 @@ def test_structured_diffusion_config_rejects_non_boolean_compile_dynamic():
|
||||
omni_config_module._DiffusionConfigProjection(diffusion_compile_dynamic="false")
|
||||
|
||||
|
||||
def test_structured_diffusion_config_rejects_non_boolean_vae_channels_last_3d():
|
||||
with pytest.raises(ValidationError, match="vae_use_channels_last_3d"):
|
||||
omni_config_module._DiffusionConfigProjection(vae_use_channels_last_3d="true")
|
||||
|
||||
|
||||
def test_from_pipeline_config_routes_vae_channels_last_3d(tmp_path):
|
||||
deploy_path = tmp_path / "dreamzero_vae_layout.yaml"
|
||||
deploy_path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"pipeline: dreamzero",
|
||||
"async_chunk: false",
|
||||
"stages:",
|
||||
" - stage_id: 0",
|
||||
" vae_use_channels_last_3d: true",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
stage = _from_pipeline_key("dreamzero", deploy_config_path=str(deploy_path)).stage_by_id(0)
|
||||
|
||||
assert stage.diffusion_config.vae_use_channels_last_3d is True
|
||||
|
||||
|
||||
def test_structured_diffusion_config_rejects_invalid_compile_granularity():
|
||||
with pytest.raises(ValidationError, match="diffusion_compile_granularity"):
|
||||
omni_config_module._DiffusionConfigProjection(diffusion_compile_granularity="block")
|
||||
|
||||
@@ -11,6 +11,85 @@ import torch
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestLTXChannelsLast3D:
|
||||
@pytest.mark.parametrize(
|
||||
("causal", "kernel_size"),
|
||||
[
|
||||
(True, 3),
|
||||
(False, 3),
|
||||
(True, (3, 1, 1)),
|
||||
(True, 1),
|
||||
],
|
||||
)
|
||||
def test_layout_preserving_causal_conv_matches_original(self, causal, kernel_size):
|
||||
from diffusers.models.autoencoders.autoencoder_kl_ltx2 import LTX2VideoCausalConv3d
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
|
||||
_enable_channels_last_3d,
|
||||
_weight_is_channels_last_3d,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
conv = LTX2VideoCausalConv3d(4, 5, kernel_size).to(dtype=torch.float32).eval()
|
||||
hidden_states = torch.randn(1, 4, 5, 6, 7, dtype=torch.float32)
|
||||
|
||||
# Pointwise Conv3d storage is valid in both contiguous formats because
|
||||
# all spatial/temporal kernel dimensions are singleton.
|
||||
if kernel_size != 1:
|
||||
assert not _weight_is_channels_last_3d(conv)
|
||||
with torch.no_grad():
|
||||
expected = conv(hidden_states.clone(), causal=causal)
|
||||
|
||||
converted, patched = _enable_channels_last_3d(conv)
|
||||
|
||||
assert (converted, patched) == (1, 1)
|
||||
assert _weight_is_channels_last_3d(conv)
|
||||
with torch.no_grad():
|
||||
actual = conv(hidden_states.clone(), causal=causal)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-4, atol=1e-4)
|
||||
|
||||
def test_temporal_pad_replicates_edges_in_channels_last_3d(self):
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
|
||||
_temporal_pad_channels_last_3d,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
hidden_states = torch.randn(1, 4, 3, 2, 2, dtype=torch.float32)
|
||||
|
||||
actual = _temporal_pad_channels_last_3d(hidden_states, left=2, right=1)
|
||||
expected = torch.cat(
|
||||
[
|
||||
hidden_states[:, :, :1].repeat(1, 1, 2, 1, 1),
|
||||
hidden_states,
|
||||
hidden_states[:, :, -1:],
|
||||
],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
assert actual.is_contiguous(memory_format=torch.channels_last_3d)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
def test_contiguous_weight_keeps_original_padding_path(self, monkeypatch):
|
||||
from diffusers.models.autoencoders.autoencoder_kl_ltx2 import LTX2VideoCausalConv3d
|
||||
|
||||
import vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 as ltx2_vae
|
||||
|
||||
conv = LTX2VideoCausalConv3d(4, 5, 3).to(dtype=torch.float32).eval()
|
||||
hidden_states = torch.randn(1, 4, 3, 2, 2, dtype=torch.float32)
|
||||
|
||||
def fail_channels_last_pad(*_args, **_kwargs):
|
||||
raise AssertionError("contiguous weights must retain the original padding path")
|
||||
|
||||
monkeypatch.setattr(ltx2_vae, "_temporal_pad_channels_last_3d", fail_channels_last_pad)
|
||||
|
||||
with torch.no_grad():
|
||||
expected = conv(hidden_states, causal=True)
|
||||
actual = ltx2_vae._layout_preserving_causal_conv_forward(conv, hidden_states, causal=True)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
class TestLTXOutputRank:
|
||||
@pytest.mark.parametrize(
|
||||
("distributed_vae_state", "expected_decode_calls"),
|
||||
|
||||
@@ -534,6 +534,18 @@ class TestDiffusionCompileConfig:
|
||||
with pytest.raises(TypeError, match="diffusion_compile_dynamic"):
|
||||
OmniDiffusionConfig(model="test", diffusion_compile_dynamic="false")
|
||||
|
||||
def test_channels_last_3d_is_opt_in_and_strict(self) -> None:
|
||||
assert OmniDiffusionConfig(model="test").vae_use_channels_last_3d is False
|
||||
assert (
|
||||
OmniDiffusionConfig.from_kwargs(
|
||||
model="test",
|
||||
vae_use_channels_last_3d=True,
|
||||
).vae_use_channels_last_3d
|
||||
is True
|
||||
)
|
||||
with pytest.raises(TypeError, match="vae_use_channels_last_3d"):
|
||||
OmniDiffusionConfig(model="test", vae_use_channels_last_3d="true")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs, feature",
|
||||
[
|
||||
|
||||
@@ -20,6 +20,7 @@ def test_default_stage_config_includes_cache_backend():
|
||||
"cache_backend": "cache_dit",
|
||||
"cache_config": '{"Fn_compute_blocks": 2}',
|
||||
"vae_use_slicing": True,
|
||||
"vae_use_channels_last_3d": True,
|
||||
"ulysses_degree": 2,
|
||||
}
|
||||
)[0]
|
||||
@@ -29,6 +30,7 @@ def test_default_stage_config_includes_cache_backend():
|
||||
assert engine_args["cache_backend"] == "cache_dit"
|
||||
assert engine_args["cache_config"]["Fn_compute_blocks"] == 2
|
||||
assert engine_args["vae_use_slicing"] is True
|
||||
assert engine_args["vae_use_channels_last_3d"] is True
|
||||
assert engine_args["parallel_config"].ulysses_degree == 2
|
||||
assert engine_args["model_stage"] == "diffusion"
|
||||
|
||||
@@ -371,6 +373,25 @@ def test_serve_cli_accepts_diffusion_compile_controls():
|
||||
assert stage_cfg["engine_args"]["diffusion_compile_dynamic"] is False
|
||||
|
||||
|
||||
def test_serve_cli_accepts_vae_channels_last_3d_opt_in():
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"serve",
|
||||
"Lightricks/LTX-2",
|
||||
"--omni",
|
||||
"--vae-use-channels-last-3d",
|
||||
]
|
||||
)
|
||||
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(args.get_explicit_kwargs_dict())[0]
|
||||
|
||||
assert args.vae_use_channels_last_3d is True
|
||||
assert stage_cfg["engine_args"]["vae_use_channels_last_3d"] is True
|
||||
|
||||
|
||||
def test_serve_cli_accepts_diffusion_attention_backend():
|
||||
"""Ensure diffusion serve CLI exposes the shorthand backend flag."""
|
||||
parser = TrackingArgumentParser()
|
||||
|
||||
Reference in new issue
Block a user