[Feat] Enable cache-dit for stable diffusion3.5 (#584)
Signed-off-by: iwzbi <wzbi@zju.edu.cn>
This commit is contained in:
1 parent
fd52647028
commit
fe509c0c10
3 files changed
+87
-13
No files matched your search
@@ -38,20 +38,14 @@ The following table shows which models are currently supported by each accelerat
|
|||||||
| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention |
|
| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention |
|
||||||
|-------|-----------------|----------|-----------|-----------|----------------|
|
|-------|-----------------|----------|-----------|-----------|----------------|
|
||||||
| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ |
|
| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ |
|
||||||
|
| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ✅ |
|
||||||
|
| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ❌ | ✅ | ✅ | ✅ |
|
||||||
|
| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ | ✅ |
|
||||||
| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ✅ |❌ | ❌ |
|
| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ✅ |❌ | ❌ |
|
||||||
| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ |✅ | - |
|
| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ |
|
||||||
| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ❌ | ✅ |✅ | - |
|
| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ |
|
||||||
=======
|
| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ |
|
||||||
| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP |
|
| **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ |
|
||||||
|-------|------------------|----------|-----------|-----------|
|
|
||||||
| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ |
|
|
||||||
| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ |
|
|
||||||
| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ |
|
|
||||||
| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ |
|
|
||||||
| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ |
|
|
||||||
| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ | ✅ |
|
|
||||||
| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ |
|
|
||||||
| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ✅ | ❌ |
|
|
||||||
|
|
||||||
### VideoGen
|
### VideoGen
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
"Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512",
|
"Qwen/Qwen-Image, Tongyi-MAI/Z-Image-Turbo, Qwen/Qwen-Image-2512",
|
||||||
)
|
)
|
||||||
parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.")
|
parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--negative_prompt", default="", help="negative prompt for classifier-free conditional guidance."
|
||||||
|
)
|
||||||
parser.add_argument("--seed", type=int, default=142, help="Random seed for deterministic results.")
|
parser.add_argument("--seed", type=int, default=142, help="Random seed for deterministic results.")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--cfg_scale",
|
"--cfg_scale",
|
||||||
@@ -29,6 +32,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=4.0,
|
default=4.0,
|
||||||
help="True classifier-free guidance scale specific to Qwen-Image.",
|
help="True classifier-free guidance scale specific to Qwen-Image.",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--guidance_scale",
|
||||||
|
type=float,
|
||||||
|
default=1.0,
|
||||||
|
help="Classifier-free guidance scale.",
|
||||||
|
)
|
||||||
parser.add_argument("--height", type=int, default=1024, help="Height of generated image.")
|
parser.add_argument("--height", type=int, default=1024, help="Height of generated image.")
|
||||||
parser.add_argument("--width", type=int, default=1024, help="Width of generated image.")
|
parser.add_argument("--width", type=int, default=1024, help="Width of generated image.")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -137,10 +146,12 @@ def main():
|
|||||||
generation_start = time.perf_counter()
|
generation_start = time.perf_counter()
|
||||||
outputs = omni.generate(
|
outputs = omni.generate(
|
||||||
args.prompt,
|
args.prompt,
|
||||||
|
negative_prompt=args.negative_prompt,
|
||||||
height=args.height,
|
height=args.height,
|
||||||
width=args.width,
|
width=args.width,
|
||||||
generator=generator,
|
generator=generator,
|
||||||
true_cfg_scale=args.cfg_scale,
|
true_cfg_scale=args.cfg_scale,
|
||||||
|
guidance_scale=args.guidance_scale,
|
||||||
num_inference_steps=args.num_inference_steps,
|
num_inference_steps=args.num_inference_steps,
|
||||||
num_outputs_per_prompt=args.num_images_per_prompt,
|
num_outputs_per_prompt=args.num_images_per_prompt,
|
||||||
)
|
)
|
||||||
|
|||||||
+69
@@ -273,6 +273,74 @@ def enable_cache_for_flux(pipeline: Any, cache_config: Any) -> Callable[[int], N
|
|||||||
raise NotImplementedError("cache-dit is not implemented for Flux pipeline.")
|
raise NotImplementedError("cache-dit is not implemented for Flux pipeline.")
|
||||||
|
|
||||||
|
|
||||||
|
def enable_cache_for_sd3(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||||
|
"""Enable cache-dit for StableDiffusion3Pipeline.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pipeline: The StableDiffusion3 pipeline instance.
|
||||||
|
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||||
|
"""
|
||||||
|
# Build DBCacheConfig for transformer
|
||||||
|
db_cache_config = _build_db_cache_config(cache_config)
|
||||||
|
|
||||||
|
calibrator = None
|
||||||
|
if cache_config.enable_taylorseer:
|
||||||
|
taylorseer_order = cache_config.taylorseer_order
|
||||||
|
calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||||
|
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||||
|
|
||||||
|
# Build ParamsModifier for transformer
|
||||||
|
modifier = ParamsModifier(
|
||||||
|
cache_config=db_cache_config,
|
||||||
|
calibrator_config=calibrator,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Enabling cache-dit on StableDiffusion3 transformer with BlockAdapter: "
|
||||||
|
f"Fn={db_cache_config.Fn_compute_blocks}, "
|
||||||
|
f"Bn={db_cache_config.Bn_compute_blocks}, "
|
||||||
|
f"W={db_cache_config.max_warmup_steps}, "
|
||||||
|
)
|
||||||
|
|
||||||
|
# Enable cache-dit using BlockAdapter for transformer
|
||||||
|
cache_dit.enable_cache(
|
||||||
|
(
|
||||||
|
BlockAdapter(
|
||||||
|
transformer=pipeline.transformer,
|
||||||
|
blocks=pipeline.transformer.transformer_blocks,
|
||||||
|
forward_pattern=ForwardPattern.Pattern_1,
|
||||||
|
params_modifiers=[modifier],
|
||||||
|
)
|
||||||
|
),
|
||||||
|
cache_config=db_cache_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||||
|
"""Refresh cache context for the transformer with new num_inference_steps.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pipeline: The LongCatImage pipeline instance.
|
||||||
|
num_inference_steps: New number of inference steps.
|
||||||
|
"""
|
||||||
|
if cache_config.scm_steps_mask_policy is None:
|
||||||
|
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
|
||||||
|
else:
|
||||||
|
cache_dit.refresh_context(
|
||||||
|
pipeline.transformer,
|
||||||
|
cache_config=DBCacheConfig().reset(
|
||||||
|
num_inference_steps=num_inference_steps,
|
||||||
|
steps_computation_mask=cache_dit.steps_mask(
|
||||||
|
mask_policy=cache_config.scm_steps_mask_policy,
|
||||||
|
total_steps=num_inference_steps,
|
||||||
|
),
|
||||||
|
steps_computation_policy=cache_config.scm_steps_policy,
|
||||||
|
),
|
||||||
|
verbose=verbose,
|
||||||
|
)
|
||||||
|
|
||||||
|
return refresh_cache_context
|
||||||
|
|
||||||
|
|
||||||
def enable_cache_for_dit(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
def enable_cache_for_dit(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||||
"""Enable cache-dit for regular single-transformer DiT models.
|
"""Enable cache-dit for regular single-transformer DiT models.
|
||||||
|
|
||||||
@@ -340,6 +408,7 @@ CUSTOM_DIT_ENABLERS.update(
|
|||||||
"FluxPipeline": enable_cache_for_flux,
|
"FluxPipeline": enable_cache_for_flux,
|
||||||
"LongcatImagePipeline": enable_cache_for_longcat_image,
|
"LongcatImagePipeline": enable_cache_for_longcat_image,
|
||||||
"LongcatImageEditPipeline": enable_cache_for_longcat_image,
|
"LongcatImageEditPipeline": enable_cache_for_longcat_image,
|
||||||
|
"StableDiffusion3Pipeline": enable_cache_for_sd3,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user