[Model] Support Flux1 Schnell (#2528)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
12378824fb
commit
d463978098
6 files changed
+49
-4
No files matched your search
@@ -47,6 +47,7 @@ th {
|
||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
||||
| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -109,6 +109,7 @@ The following tables show which models support each feature:
|
||||
|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||
| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **FLUX.1-dev** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
|
||||
| **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
|
||||
| **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
|
||||
| **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **FLUX.2-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
|
||||
@@ -29,7 +29,8 @@ This folder provides several entrypoints for experimenting with text-to-image di
|
||||
| `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
|
||||
| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
|
||||
| `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
|
||||
| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 77.6 | 31.4 |
|
||||
| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
|
||||
| `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
|
||||
| `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
|
||||
| `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
|
||||
| `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for Flux1 Schnell."""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-schnell"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux_schnell_text_to_image():
|
||||
"""Test FLUX.1-schnell text-to-image generation."""
|
||||
omni = Omni(model=MODEL)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=["A photo of a cat sitting on a laptop"],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
images = omni_outputs[0].images
|
||||
assert len(images) == 1
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
@@ -526,10 +526,10 @@ class FluxTransformer2DModel(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig = None,
|
||||
od_config: OmniDiffusionConfig | None = None,
|
||||
patch_size: int = 1,
|
||||
in_channels: int = 64,
|
||||
out_channels: int = None,
|
||||
out_channels: int | None = None,
|
||||
num_layers: int = 19,
|
||||
num_single_layers: int = 38,
|
||||
attention_head_dim: int = 128,
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm_omni.diffusion.models.flux.flux_pipeline_mixin import FluxPipelineMixi
|
||||
from vllm_omni.diffusion.models.t5_encoder import T5EncoderModel
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -106,7 +107,11 @@ class FluxPipeline(nn.Module, FluxPipelineMixin, CFGParallelMixin, DiffusionPipe
|
||||
self.vae = AutoencoderKL.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to(
|
||||
self.device
|
||||
)
|
||||
self.transformer = FluxTransformer2DModel(od_config=od_config, quant_config=od_config.quantization_config)
|
||||
|
||||
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, FluxTransformer2DModel)
|
||||
self.transformer = FluxTransformer2DModel(
|
||||
**transformer_kwargs, od_config=od_config, quant_config=od_config.quantization_config
|
||||
)
|
||||
|
||||
self.tokenizer = CLIPTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
self.tokenizer_2 = T5TokenizerFast.from_pretrained(
|
||||
|
||||
Reference in new issue
Block a user