Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c8dfe2344 |
No files matched your search
@@ -28,7 +28,6 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b
|
|||||||
- Qwen/Qwen-Image
|
- Qwen/Qwen-Image
|
||||||
- Tongyi-MAI/Z-Image-Turbo
|
- Tongyi-MAI/Z-Image-Turbo
|
||||||
- Wan2.2-I2V-A14B-Diffusers
|
- Wan2.2-I2V-A14B-Diffusers
|
||||||
- Lightricks/LTX-2.5-Diffusers (source-preview dependencies)
|
|
||||||
|
|
||||||
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
||||||
Please consider file an issue.
|
Please consider file an issue.
|
||||||
@@ -117,36 +116,6 @@ The model loading and inference strictly follows the diffusers library, and they
|
|||||||
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
||||||
Some particular examples are below.
|
Some particular examples are below.
|
||||||
|
|
||||||
#### LTX-2.5 (source preview)
|
|
||||||
|
|
||||||
Use the gated Diffusers checkpoint
|
|
||||||
[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers);
|
|
||||||
[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains
|
|
||||||
component weights and is not directly loadable by this backend. Accept the
|
|
||||||
model terms, authenticate, and install the source-preview dependencies:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
hf auth login
|
|
||||||
python -m pip install --upgrade 'transformers>=5.10.1' \
|
|
||||||
'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20'
|
|
||||||
```
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve Lightricks/LTX-2.5-Diffusers \
|
|
||||||
--omni \
|
|
||||||
--diffusion-load-format diffusers \
|
|
||||||
--dtype bfloat16 \
|
|
||||||
--enable-cpu-offload \
|
|
||||||
--vae-use-tiling
|
|
||||||
```
|
|
||||||
|
|
||||||
The converted repository loads the distilled transformer. When no `sigmas`,
|
|
||||||
`timesteps`, or `num_inference_steps` is supplied, the adapter applies the
|
|
||||||
official eight-sigma unguided schedule and returns synchronized 48 kHz audio.
|
|
||||||
Support is limited to one-stage distilled T2V; `transformer_full`, two-stage
|
|
||||||
upsampling, image/video conditioning, and the diffusion video decoder are not
|
|
||||||
supported. The general Diffusers backend [limitations](#limitations) apply.
|
|
||||||
|
|
||||||
#### Wan Series
|
#### Wan Series
|
||||||
|
|
||||||
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b
|
|||||||
- Qwen/Qwen-Image
|
- Qwen/Qwen-Image
|
||||||
- Tongyi-MAI/Z-Image-Turbo
|
- Tongyi-MAI/Z-Image-Turbo
|
||||||
- Wan2.2-I2V-A14B-Diffusers
|
- Wan2.2-I2V-A14B-Diffusers
|
||||||
- Lightricks/LTX-2.5-Diffusers (source-preview dependencies)
|
|
||||||
|
|
||||||
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
||||||
Please consider file an issue or submit a PR to fix.
|
Please consider file an issue or submit a PR to fix.
|
||||||
@@ -119,36 +118,6 @@ The model loading and inference strictly follows the diffusers library, and they
|
|||||||
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
||||||
Some particular examples are below.
|
Some particular examples are below.
|
||||||
|
|
||||||
#### LTX-2.5 (source preview)
|
|
||||||
|
|
||||||
Use the gated Diffusers checkpoint
|
|
||||||
[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers);
|
|
||||||
[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains
|
|
||||||
component weights and is not directly loadable by this backend. Accept the
|
|
||||||
model terms, authenticate, and install the source-preview dependencies:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
hf auth login
|
|
||||||
python -m pip install --upgrade 'transformers>=5.10.1' \
|
|
||||||
'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20'
|
|
||||||
```
|
|
||||||
|
|
||||||
```bash
|
|
||||||
vllm serve Lightricks/LTX-2.5-Diffusers \
|
|
||||||
--omni \
|
|
||||||
--diffusion-load-format diffusers \
|
|
||||||
--dtype bfloat16 \
|
|
||||||
--enable-cpu-offload \
|
|
||||||
--vae-use-tiling
|
|
||||||
```
|
|
||||||
|
|
||||||
The converted repository loads the distilled transformer. When no `sigmas`,
|
|
||||||
`timesteps`, or `num_inference_steps` is supplied, the adapter applies the
|
|
||||||
official eight-sigma unguided schedule and returns synchronized 48 kHz audio.
|
|
||||||
Support is limited to one-stage distilled T2V; `transformer_full`, two-stage
|
|
||||||
upsampling, image/video conditioning, and the diffusion video decoder are not
|
|
||||||
supported. The general Diffusers backend [limitations](#limitations) apply.
|
|
||||||
|
|
||||||
#### Wan Series
|
#### Wan Series
|
||||||
|
|
||||||
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
||||||
|
|||||||
@@ -17,10 +17,6 @@ from vllm_omni.diffusion.data import (
|
|||||||
OmniDiffusionConfig,
|
OmniDiffusionConfig,
|
||||||
)
|
)
|
||||||
from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline
|
from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline
|
||||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
|
|
||||||
LTX25_DISTILLED_SIGMAS,
|
|
||||||
get_pipeline_utils,
|
|
||||||
)
|
|
||||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
@@ -184,32 +180,6 @@ class TestPipelineArgumentsHandling:
|
|||||||
with pytest.raises(NotImplementedError):
|
with pytest.raises(NotImplementedError):
|
||||||
DiffusersAdapterPipeline(od_config=od_config)
|
DiffusersAdapterPipeline(od_config=od_config)
|
||||||
|
|
||||||
def test_ltx25_original_repo_rejected_before_loading(self, mocker):
|
|
||||||
mock_from_pretrained = mocker.patch(
|
|
||||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained"
|
|
||||||
)
|
|
||||||
adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5"))
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="Lightricks/LTX-2.5-Diffusers"):
|
|
||||||
adapter.load_weights()
|
|
||||||
|
|
||||||
mock_from_pretrained.assert_not_called()
|
|
||||||
|
|
||||||
def test_ltx25_requires_upstream_diffusers_feature(self, mocker):
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers",
|
|
||||||
return_value=False,
|
|
||||||
)
|
|
||||||
mock_from_pretrained = mocker.patch(
|
|
||||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained"
|
|
||||||
)
|
|
||||||
adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5-Diffusers"))
|
|
||||||
|
|
||||||
with pytest.raises(ImportError, match="7564fb016dabda0c943416190fc92398c50b1b20"):
|
|
||||||
adapter.load_weights()
|
|
||||||
|
|
||||||
mock_from_pretrained.assert_not_called()
|
|
||||||
|
|
||||||
def test_adapter_load_weights_injects_supported_quantization_config(self, mocker):
|
def test_adapter_load_weights_injects_supported_quantization_config(self, mocker):
|
||||||
(
|
(
|
||||||
FakePipelineQuantizationConfig,
|
FakePipelineQuantizationConfig,
|
||||||
@@ -404,28 +374,6 @@ class TestPipelineArgumentsHandling:
|
|||||||
assert isinstance(output, DiffusionOutput)
|
assert isinstance(output, DiffusionOutput)
|
||||||
assert output.output == raw_output
|
assert output.output == raw_output
|
||||||
|
|
||||||
def test_ltx25_preserves_video_and_singular_audio_output(self, tmp_path):
|
|
||||||
frames = torch.zeros(1, 2, 3, 4, 3)
|
|
||||||
audio = torch.zeros(1, 48000)
|
|
||||||
(tmp_path / "model_index.json").write_text(
|
|
||||||
'{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}',
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
od_config = _make_od_config(model=str(tmp_path))
|
|
||||||
adapter = DiffusersAdapterPipeline(od_config=od_config)
|
|
||||||
adapter._pipeline = SimpleNamespace(vocoder=SimpleNamespace(config=SimpleNamespace(output_sampling_rate=48000)))
|
|
||||||
adapter._pipeline_utils = get_pipeline_utils("LTX2Pipeline")
|
|
||||||
|
|
||||||
output = adapter._wrap_output(SimpleNamespace(frames=frames, audio=audio))
|
|
||||||
|
|
||||||
assert isinstance(output.output, dict)
|
|
||||||
assert output.output["payload"]["video"] is frames
|
|
||||||
assert output.output["payload"]["audio"] is audio
|
|
||||||
assert output.output["metadata"] == {"audio": {"sample_rate": 48000}}
|
|
||||||
|
|
||||||
frames_only = adapter._wrap_output(SimpleNamespace(frames=frames, audio=None))
|
|
||||||
assert frames_only.output is frames
|
|
||||||
|
|
||||||
def test_adapter_build_call_kwargs(self, mocker):
|
def test_adapter_build_call_kwargs(self, mocker):
|
||||||
class MockPipeline:
|
class MockPipeline:
|
||||||
def __call__(
|
def __call__(
|
||||||
@@ -681,97 +629,6 @@ class TestPipelineArgumentsHandling:
|
|||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
pipeline.forward(DiffusionRequestBatch(requests=[problematic_request]))
|
pipeline.forward(DiffusionRequestBatch(requests=[problematic_request]))
|
||||||
|
|
||||||
def test_ltx25_uses_distilled_sampling_defaults_from_renamed_local_directory(self, mocker, tmp_path):
|
|
||||||
class LTX2Pipeline:
|
|
||||||
def __call__(
|
|
||||||
self,
|
|
||||||
prompt=None,
|
|
||||||
num_inference_steps=None,
|
|
||||||
sigmas=None,
|
|
||||||
timesteps=None,
|
|
||||||
guidance_scale=None,
|
|
||||||
audio_guidance_scale=None,
|
|
||||||
stg_scale=None,
|
|
||||||
audio_stg_scale=None,
|
|
||||||
modality_scale=None,
|
|
||||||
audio_modality_scale=None,
|
|
||||||
guidance_rescale=None,
|
|
||||||
audio_guidance_rescale=None,
|
|
||||||
num_frames=None,
|
|
||||||
frame_rate=None,
|
|
||||||
output_type=None,
|
|
||||||
generator=None,
|
|
||||||
):
|
|
||||||
return None
|
|
||||||
|
|
||||||
def to(self, device):
|
|
||||||
return self
|
|
||||||
|
|
||||||
# Local model directories do not retain their Hub repository name.
|
|
||||||
# Detect 2.5 from its pipeline component metadata instead.
|
|
||||||
(tmp_path / "model_index.json").write_text(
|
|
||||||
'{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}',
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
enable_tiling = mocker.Mock()
|
|
||||||
loaded_pipeline = LTX2Pipeline()
|
|
||||||
loaded_pipeline.vae = SimpleNamespace(enable_tiling=enable_tiling)
|
|
||||||
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers",
|
|
||||||
return_value=True,
|
|
||||||
)
|
|
||||||
mocker.patch(
|
|
||||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained",
|
|
||||||
return_value=loaded_pipeline,
|
|
||||||
)
|
|
||||||
adapter = DiffusersAdapterPipeline(
|
|
||||||
od_config=_make_od_config(
|
|
||||||
model=str(tmp_path),
|
|
||||||
vae_use_tiling=True,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
adapter.load_weights()
|
|
||||||
enable_tiling.assert_called_once_with()
|
|
||||||
|
|
||||||
sampling = OmniDiffusionSamplingParams(
|
|
||||||
num_frames=121,
|
|
||||||
frame_rate=24.0,
|
|
||||||
seed=42,
|
|
||||||
generator_device="cpu",
|
|
||||||
)
|
|
||||||
kwargs = adapter._build_call_kwargs(DiffusionRequestBatch(requests=[_make_request(sampling_params=sampling)]))
|
|
||||||
|
|
||||||
assert kwargs["sigmas"] == LTX25_DISTILLED_SIGMAS
|
|
||||||
assert kwargs["num_inference_steps"] == 8
|
|
||||||
assert kwargs["guidance_scale"] == 1.0
|
|
||||||
assert kwargs["audio_guidance_scale"] == 1.0
|
|
||||||
assert kwargs["stg_scale"] == 0.0
|
|
||||||
assert kwargs["audio_stg_scale"] == 0.0
|
|
||||||
assert kwargs["modality_scale"] == 1.0
|
|
||||||
assert kwargs["audio_modality_scale"] == 1.0
|
|
||||||
assert kwargs["guidance_rescale"] == 0.0
|
|
||||||
assert kwargs["audio_guidance_rescale"] == 0.0
|
|
||||||
|
|
||||||
explicit_steps = OmniDiffusionSamplingParams(num_inference_steps=12, generator_device="cpu")
|
|
||||||
step_kwargs = adapter._build_call_kwargs(
|
|
||||||
DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_steps)])
|
|
||||||
)
|
|
||||||
assert step_kwargs["num_inference_steps"] == 12
|
|
||||||
assert "sigmas" not in step_kwargs
|
|
||||||
|
|
||||||
custom_sigmas = [1.0, 0.5]
|
|
||||||
explicit_schedule = OmniDiffusionSamplingParams(
|
|
||||||
num_inference_steps=2,
|
|
||||||
sigmas=custom_sigmas,
|
|
||||||
generator_device="cpu",
|
|
||||||
)
|
|
||||||
schedule_kwargs = adapter._build_call_kwargs(
|
|
||||||
DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_schedule)])
|
|
||||||
)
|
|
||||||
assert schedule_kwargs["num_inference_steps"] == 2
|
|
||||||
assert schedule_kwargs["sigmas"] is custom_sigmas
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import torch.nn as nn
|
|||||||
|
|
||||||
import vllm_omni.diffusion.compile as compile_module
|
import vllm_omni.diffusion.compile as compile_module
|
||||||
from vllm_omni.diffusion.compile import regionally_compile
|
from vllm_omni.diffusion.compile import regionally_compile
|
||||||
|
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||||
|
|
||||||
@@ -76,6 +77,37 @@ def test_regionally_compile_does_not_partially_mutate_on_setup_failure(monkeypat
|
|||||||
assert [block.forward.__func__ for block in model.transformer_blocks] == original_forwards
|
assert [block.forward.__func__ for block in model.transformer_blocks] == original_forwards
|
||||||
|
|
||||||
|
|
||||||
|
def test_regionally_compile_keeps_hook_dispatch_outside_compiled_graph(monkeypatch):
|
||||||
|
model = _ModelWithWrappedRepeatedBlocks()
|
||||||
|
block = model.transformer_blocks[0]
|
||||||
|
registry = HookRegistry.get_or_create(block)
|
||||||
|
hook = ModelHook()
|
||||||
|
registry.register_hook("test", hook)
|
||||||
|
|
||||||
|
wrapped_forward = block.forward
|
||||||
|
original_forward = block._omni_original_forward
|
||||||
|
compile_calls = []
|
||||||
|
|
||||||
|
def _compile(fn, *args, **kwargs):
|
||||||
|
compile_calls.append(fn)
|
||||||
|
|
||||||
|
def _compiled(*fn_args, **fn_kwargs):
|
||||||
|
return f"compiled:{fn(*fn_args, **fn_kwargs)}"
|
||||||
|
|
||||||
|
return _compiled
|
||||||
|
|
||||||
|
monkeypatch.setattr(compile_module.torch, "compile", _compile)
|
||||||
|
|
||||||
|
regionally_compile(model)
|
||||||
|
|
||||||
|
assert compile_calls[0] is original_forward
|
||||||
|
assert compile_calls[0] is not wrapped_forward
|
||||||
|
assert block.forward is wrapped_forward
|
||||||
|
assert block._omni_original_forward is not original_forward
|
||||||
|
assert hook.fn_ref.original_forward is block._omni_original_forward
|
||||||
|
assert block("ok") == "compiled:ok"
|
||||||
|
|
||||||
|
|
||||||
def test_compiled_block_preserves_forward_signature_for_inspection(monkeypatch):
|
def test_compiled_block_preserves_forward_signature_for_inspection(monkeypatch):
|
||||||
"""cache-dit matches blocks via inspect.signature(block.forward).
|
"""cache-dit matches blocks via inspect.signature(block.forward).
|
||||||
|
|
||||||
|
|||||||
@@ -56,26 +56,50 @@ def regionally_compile(
|
|||||||
# Build all compiled callables before mutating the model. This keeps setup
|
# Build all compiled callables before mutating the model. This keeps setup
|
||||||
# failures atomic: callers can safely continue with the uncompiled model if
|
# failures atomic: callers can safely continue with the uncompiled model if
|
||||||
# torch.compile raises synchronously for any repeated block.
|
# torch.compile raises synchronously for any repeated block.
|
||||||
compiled_forwards: list[tuple[nn.Module, Any]] = []
|
# Keep the pre-hook callable separately. DLO and the ordinary layerwise
|
||||||
|
# offloader install a HookRegistry wrapper in ``module.forward``. That
|
||||||
|
# wrapper performs stream/event synchronization and storage rebinding, so
|
||||||
|
# compiling it pulls offload control flow into the graph. Compile the
|
||||||
|
# original block compute instead and leave the wrapper outside the graph.
|
||||||
|
compiled_forwards: list[tuple[nn.Module, Any | None, Any]] = []
|
||||||
for name, submod in model.named_modules():
|
for name, submod in model.named_modules():
|
||||||
if _matches_repeated_block(name, submod, repeated_blocks, repeated_block_attrs):
|
if _matches_repeated_block(name, submod, repeated_blocks, repeated_block_attrs):
|
||||||
# Compile the block compute while keeping nn.Module.__call__ hooks
|
# Compile the block compute while keeping nn.Module.__call__ hooks
|
||||||
# outside the compiled graph. NOTE: anything that wraps this
|
# outside the compiled graph. If a HookRegistry is already
|
||||||
# callable must stay signature-transparent — cache-dit's
|
# installed, ``submod.forward`` is the hook dispatcher and the
|
||||||
# BlockAdapter matches blocks by inspecting forward's parameter
|
# original callable is stored in ``_omni_original_forward``.
|
||||||
# names and return annotation, which torch.compile preserves.
|
# NOTE: anything that wraps this callable must stay
|
||||||
|
# signature-transparent — cache-dit's BlockAdapter matches blocks
|
||||||
|
# by inspecting forward's parameter names and return annotation,
|
||||||
|
# which torch.compile preserves.
|
||||||
|
original_forward = getattr(submod, "_omni_original_forward", None)
|
||||||
|
forward_to_compile = original_forward if original_forward is not None else submod.forward
|
||||||
compiled_forwards.append(
|
compiled_forwards.append(
|
||||||
(
|
(
|
||||||
submod,
|
submod,
|
||||||
torch.compile(submod.forward, *compile_args, **compile_kwargs),
|
original_forward,
|
||||||
|
torch.compile(forward_to_compile, *compile_args, **compile_kwargs),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if not compiled_forwards:
|
if not compiled_forwards:
|
||||||
logger.warning(f"Regional compilation skipped because {repeated_blocks} classes are not found in the model.")
|
logger.warning(f"Regional compilation skipped because {repeated_blocks} classes are not found in the model.")
|
||||||
else:
|
else:
|
||||||
for submod, compiled_forward in compiled_forwards:
|
for submod, original_forward, compiled_forward in compiled_forwards:
|
||||||
submod.forward = compiled_forward
|
if original_forward is None:
|
||||||
|
submod.forward = compiled_forward
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Keep HookRegistry's dispatcher as ``forward`` and replace only
|
||||||
|
# the callable it dispatches to. Hooks such as MagCache retain a
|
||||||
|
# reference to the original callable as well, so update those
|
||||||
|
# references when they still point at the pre-compile function.
|
||||||
|
submod._omni_original_forward = compiled_forward
|
||||||
|
registry = getattr(submod, "_hook_registry", None)
|
||||||
|
for hook in getattr(registry, "_hooks", {}).values():
|
||||||
|
fn_ref = getattr(hook, "fn_ref", None)
|
||||||
|
if fn_ref is not None and fn_ref.original_forward is original_forward:
|
||||||
|
fn_ref.original_forward = compiled_forward
|
||||||
logger.info(
|
logger.info(
|
||||||
"Regional compilation applied to %d module(s) for repeated blocks %s.",
|
"Regional compilation applied to %d module(s) for repeated blocks %s.",
|
||||||
len(compiled_forwards),
|
len(compiled_forwards),
|
||||||
|
|||||||
@@ -23,11 +23,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
|
|
||||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
|
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils
|
||||||
BasePipelineUtils,
|
|
||||||
get_pipeline_utils,
|
|
||||||
validate_model_compatibility,
|
|
||||||
)
|
|
||||||
from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import (
|
from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import (
|
||||||
apply_diffusers_quantization_config,
|
apply_diffusers_quantization_config,
|
||||||
convert_diffusers_quantization_config,
|
convert_diffusers_quantization_config,
|
||||||
@@ -114,7 +110,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
|||||||
|
|
||||||
model_id = self.od_config.model
|
model_id = self.od_config.model
|
||||||
dtype = self.od_config.dtype
|
dtype = self.od_config.dtype
|
||||||
validate_model_compatibility(self.od_config)
|
|
||||||
|
|
||||||
load_kwargs = {
|
load_kwargs = {
|
||||||
"torch_dtype": dtype,
|
"torch_dtype": dtype,
|
||||||
@@ -135,8 +130,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
|||||||
logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}")
|
logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}")
|
||||||
|
|
||||||
self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)
|
self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)
|
||||||
if pipeline_class_name is None:
|
|
||||||
self._pipeline_utils = get_pipeline_utils(self._pipeline.__class__.__name__)
|
|
||||||
self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config)
|
self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config)
|
||||||
|
|
||||||
self._pipeline.to(self.device)
|
self._pipeline.to(self.device)
|
||||||
@@ -153,14 +146,14 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
|||||||
# VAE slicing and tiling: try-catch because not all models have VAE
|
# VAE slicing and tiling: try-catch because not all models have VAE
|
||||||
if self.od_config.vae_use_slicing:
|
if self.od_config.vae_use_slicing:
|
||||||
try:
|
try:
|
||||||
self._pipeline_utils.enable_vae_optimization(self._pipeline, "slicing")
|
self._pipeline.enable_vae_slicing()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Failed to enable VAE slicing for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
f"Failed to enable VAE slicing for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
||||||
)
|
)
|
||||||
if self.od_config.vae_use_tiling:
|
if self.od_config.vae_use_tiling:
|
||||||
try:
|
try:
|
||||||
self._pipeline_utils.enable_vae_optimization(self._pipeline, "tiling")
|
self._pipeline.enable_vae_tiling()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Failed to enable VAE tiling for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
f"Failed to enable VAE tiling for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
||||||
@@ -400,8 +393,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
|||||||
else:
|
else:
|
||||||
kwargs["generator"] = torch.Generator(device=sampling.generator_device)
|
kwargs["generator"] = torch.Generator(device=sampling.generator_device)
|
||||||
|
|
||||||
self._pipeline_utils.update_call_kwargs(self.od_config, kwargs)
|
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Calling diffusers pipeline with kwargs: %s", DiffusersAdapterPipeline._summarize_call_kwargs_value(kwargs)
|
"Calling diffusers pipeline with kwargs: %s", DiffusersAdapterPipeline._summarize_call_kwargs_value(kwargs)
|
||||||
)
|
)
|
||||||
@@ -506,8 +497,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
|||||||
"""
|
"""
|
||||||
from vllm_omni.diffusion.data import DiffusionOutput
|
from vllm_omni.diffusion.data import DiffusionOutput
|
||||||
|
|
||||||
output = self._pipeline_utils.normalize_output(self._pipeline, self.od_config, output)
|
|
||||||
|
|
||||||
if hasattr(output, "images"):
|
if hasattr(output, "images"):
|
||||||
# Preserve diffusers image format (`output_type`)
|
# Preserve diffusers image format (`output_type`)
|
||||||
return DiffusionOutput(output=output.images)
|
return DiffusionOutput(output=output.images)
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import inspect
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||||
@@ -12,74 +8,6 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
|||||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
LTX25_ORIGINAL_MODEL_ID = "Lightricks/LTX-2.5"
|
|
||||||
LTX25_DIFFUSERS_MODEL_ID = "Lightricks/LTX-2.5-Diffusers"
|
|
||||||
LTX25_DIFFUSERS_COMMIT = "7564fb016dabda0c943416190fc92398c50b1b20"
|
|
||||||
|
|
||||||
# Diffusers' public LTX-2.5 schedule intentionally excludes the scheduler's
|
|
||||||
# terminal zero. FlowMatchEulerDiscreteScheduler appends that internally.
|
|
||||||
LTX25_DISTILLED_SIGMAS = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875]
|
|
||||||
|
|
||||||
_LTX25_DISTILLED_GUIDANCE_DEFAULTS: dict[str, float] = {
|
|
||||||
"guidance_scale": 1.0,
|
|
||||||
"audio_guidance_scale": 1.0,
|
|
||||||
"stg_scale": 0.0,
|
|
||||||
"audio_stg_scale": 0.0,
|
|
||||||
"modality_scale": 1.0,
|
|
||||||
"audio_modality_scale": 1.0,
|
|
||||||
"guidance_rescale": 0.0,
|
|
||||||
"audio_guidance_rescale": 0.0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _supports_ltx25_diffusers() -> bool:
|
|
||||||
"""Check the standard-pipeline capabilities added with LTX-2.5."""
|
|
||||||
try:
|
|
||||||
import diffusers
|
|
||||||
import transformers
|
|
||||||
|
|
||||||
pipeline_class = diffusers.LTX2Pipeline
|
|
||||||
init_parameters = inspect.signature(pipeline_class.__init__).parameters
|
|
||||||
return (
|
|
||||||
"duration_head" in init_parameters
|
|
||||||
and "prompt_enhancer" in init_parameters
|
|
||||||
and hasattr(transformers, "Gemma4UnifiedForConditionalGeneration")
|
|
||||||
)
|
|
||||||
except (AttributeError, ImportError, RuntimeError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_ltx25_diffusers_model(od_config: OmniDiffusionConfig) -> bool:
|
|
||||||
"""Return whether config targets the official converted LTX-2.5 repo."""
|
|
||||||
normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower()
|
|
||||||
if normalized_model == LTX25_DIFFUSERS_MODEL_ID.lower():
|
|
||||||
return True
|
|
||||||
|
|
||||||
# A downloaded checkpoint may be served from any directory name. The
|
|
||||||
# duration-head component is the pipeline-level marker introduced for
|
|
||||||
# LTX-2.5; older LTX-2.0/2.3 indexes do not contain it.
|
|
||||||
model_path = Path(str(od_config.model)).expanduser()
|
|
||||||
subfolder = od_config.diffusers_load_kwargs.get("subfolder")
|
|
||||||
if subfolder is not None:
|
|
||||||
model_path /= str(subfolder)
|
|
||||||
try:
|
|
||||||
with (model_path / "model_index.json").open(encoding="utf-8") as model_index_file:
|
|
||||||
model_index = json.load(model_index_file)
|
|
||||||
except (OSError, TypeError, ValueError):
|
|
||||||
return False
|
|
||||||
if not isinstance(model_index, dict):
|
|
||||||
return False
|
|
||||||
|
|
||||||
duration_head = model_index.get("duration_head")
|
|
||||||
return (
|
|
||||||
model_index.get("_class_name") == "LTX2Pipeline"
|
|
||||||
and isinstance(duration_head, list)
|
|
||||||
and bool(duration_head)
|
|
||||||
and duration_head[0] is not None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class BasePipelineUtils:
|
class BasePipelineUtils:
|
||||||
"""No-op hooks for pipeline-specific diffusers adapter behavior."""
|
"""No-op hooks for pipeline-specific diffusers adapter behavior."""
|
||||||
@@ -90,112 +18,9 @@ class BasePipelineUtils:
|
|||||||
def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None:
|
def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None:
|
|
||||||
getattr(pipeline, f"enable_vae_{optimization}")()
|
|
||||||
|
|
||||||
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def update_call_kwargs(
|
|
||||||
self,
|
|
||||||
od_config: OmniDiffusionConfig,
|
|
||||||
call_kwargs: dict[str, Any],
|
|
||||||
) -> None:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def normalize_output(
|
|
||||||
self,
|
|
||||||
pipeline: DiffusionPipeline,
|
|
||||||
od_config: OmniDiffusionConfig,
|
|
||||||
output: Any,
|
|
||||||
) -> Any:
|
|
||||||
return output
|
|
||||||
|
|
||||||
|
|
||||||
def validate_model_compatibility(od_config: OmniDiffusionConfig) -> None:
|
|
||||||
"""Reject known model/dependency combinations before loading weights."""
|
|
||||||
normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower()
|
|
||||||
if normalized_model == LTX25_ORIGINAL_MODEL_ID.lower():
|
|
||||||
raise ValueError(
|
|
||||||
f"{LTX25_ORIGINAL_MODEL_ID} is a component-weight repository without a Diffusers "
|
|
||||||
f"model_index.json. Use the official converted repository {LTX25_DIFFUSERS_MODEL_ID} "
|
|
||||||
"with --diffusion-load-format diffusers."
|
|
||||||
)
|
|
||||||
|
|
||||||
if is_ltx25_diffusers_model(od_config) and not _supports_ltx25_diffusers():
|
|
||||||
raise ImportError(
|
|
||||||
f"{LTX25_DIFFUSERS_MODEL_ID} requires unreleased LTX-2.5 support from Diffusers and "
|
|
||||||
"Gemma 4 Unified support from Transformers. Install the source-preview dependencies with "
|
|
||||||
"`python -m pip install --upgrade 'transformers>=5.10.1' "
|
|
||||||
f"'diffusers @ git+https://github.com/huggingface/diffusers.git@{LTX25_DIFFUSERS_COMMIT}'`."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class LTX2PipelineUtils(BasePipelineUtils):
|
|
||||||
"""Diffusers call policy for the official default LTX-2.5 checkpoint."""
|
|
||||||
|
|
||||||
def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None:
|
|
||||||
pipeline_method = getattr(pipeline, f"enable_vae_{optimization}", None)
|
|
||||||
if callable(pipeline_method):
|
|
||||||
pipeline_method()
|
|
||||||
else:
|
|
||||||
getattr(pipeline.vae, f"enable_{optimization}")()
|
|
||||||
|
|
||||||
def update_call_kwargs(
|
|
||||||
self,
|
|
||||||
od_config: OmniDiffusionConfig,
|
|
||||||
call_kwargs: dict[str, Any],
|
|
||||||
) -> None:
|
|
||||||
if not is_ltx25_diffusers_model(od_config):
|
|
||||||
return
|
|
||||||
|
|
||||||
has_sigmas = call_kwargs.get("sigmas") is not None
|
|
||||||
has_timesteps = call_kwargs.get("timesteps") is not None
|
|
||||||
has_step_count = call_kwargs.get("num_inference_steps") is not None
|
|
||||||
if not has_sigmas and not has_timesteps:
|
|
||||||
if has_step_count:
|
|
||||||
logger.warning(
|
|
||||||
"LTX-2.5 distilled inference received num_inference_steps without sigmas. "
|
|
||||||
"Diffusers will use a generic schedule; omit num_inference_steps to use the "
|
|
||||||
"official distilled schedule."
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
call_kwargs["sigmas"] = list(LTX25_DISTILLED_SIGMAS)
|
|
||||||
call_kwargs["num_inference_steps"] = len(LTX25_DISTILLED_SIGMAS)
|
|
||||||
|
|
||||||
for key, value in _LTX25_DISTILLED_GUIDANCE_DEFAULTS.items():
|
|
||||||
if call_kwargs.get(key) is None:
|
|
||||||
call_kwargs[key] = value
|
|
||||||
|
|
||||||
def normalize_output(
|
|
||||||
self,
|
|
||||||
pipeline: DiffusionPipeline,
|
|
||||||
od_config: OmniDiffusionConfig,
|
|
||||||
output: Any,
|
|
||||||
) -> Any:
|
|
||||||
if not is_ltx25_diffusers_model(od_config):
|
|
||||||
return output
|
|
||||||
|
|
||||||
frames = getattr(output, "frames", None)
|
|
||||||
audio = getattr(output, "audio", None)
|
|
||||||
if frames is None or audio is None:
|
|
||||||
return output
|
|
||||||
|
|
||||||
envelope: dict[str, Any] = {"payload": {"video": frames, "audio": audio}}
|
|
||||||
config = getattr(getattr(pipeline, "vocoder", None), "config", None)
|
|
||||||
value = (
|
|
||||||
config.get("output_sampling_rate")
|
|
||||||
if hasattr(config, "get")
|
|
||||||
else getattr(config, "output_sampling_rate", None)
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
sample_rate = None if isinstance(value, bool) else int(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
sample_rate = None
|
|
||||||
if sample_rate is not None and sample_rate > 0:
|
|
||||||
envelope["metadata"] = {"audio": {"sample_rate": sample_rate}}
|
|
||||||
return envelope
|
|
||||||
|
|
||||||
|
|
||||||
class WanPipelineUtils(BasePipelineUtils):
|
class WanPipelineUtils(BasePipelineUtils):
|
||||||
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
|
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
|
||||||
@@ -224,7 +49,6 @@ class WanPipelineUtils(BasePipelineUtils):
|
|||||||
|
|
||||||
|
|
||||||
PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
|
PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
|
||||||
"LTX2Pipeline": LTX2PipelineUtils,
|
|
||||||
"WanPipeline": WanPipelineUtils,
|
"WanPipeline": WanPipelineUtils,
|
||||||
"WanImageToVideoPipeline": WanPipelineUtils,
|
"WanImageToVideoPipeline": WanPipelineUtils,
|
||||||
"WanVACEPipeline": WanPipelineUtils,
|
"WanVACEPipeline": WanPipelineUtils,
|
||||||
|
|||||||
Reference in new issue
Block a user