[Feature] Support DiT Layerwise (Blockwise) CPU Offloading (#858)
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
14 files changed
+613
-41
No files matched your search
@@ -295,6 +295,17 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable CPU offloading for diffusion models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-layerwise-offload",
|
||||
action="store_true",
|
||||
help="Enable layerwise (blockwise) offloading on DiT modules.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-num-gpu-layers",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of ready layers (blocks) to keep on GPU during generation.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -350,6 +361,8 @@ def main():
|
||||
# Initialize Omni with appropriate pipeline
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
cache_backend=args.cache_backend,
|
||||
|
||||
@@ -74,6 +74,17 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable CPU offloading for diffusion models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-layerwise-offload",
|
||||
action="store_true",
|
||||
help="Enable layerwise (blockwise) offloading on DiT modules.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-num-gpu-layers",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of ready layers (blocks) to keep on GPU during generation.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -112,6 +123,8 @@ def main():
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
|
||||
@@ -107,6 +107,17 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable CPU offloading for diffusion models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-layerwise-offload",
|
||||
action="store_true",
|
||||
help="Enable layerwise (blockwise) offloading on DiT modules.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-num-gpu-layers",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of ready layers (blocks) to keep on GPU during generation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tensor_parallel_size",
|
||||
type=int,
|
||||
@@ -172,6 +183,8 @@ def main():
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
cache_backend=args.cache_backend,
|
||||
|
||||
@@ -79,6 +79,17 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable CPU offloading for diffusion models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-layerwise-offload",
|
||||
action="store_true",
|
||||
help="Enable layerwise (blockwise) offloading on DiT modules.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-num-gpu-layers",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of ready layers (blocks) to keep on GPU during generation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ulysses_degree",
|
||||
type=int,
|
||||
@@ -128,6 +139,8 @@ def main():
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
|
||||
Reference in new issue
Block a user