[WAN_S2V] Enable HSDP for wan s2v (#4458)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
1 parent
92f7208559
commit
60ccadfbb6
5 files changed
+384
-14
No files matched your search
@@ -172,6 +172,29 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of GPUs used for ulysses sequence parallelism.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-hsdp",
|
||||
action="store_true",
|
||||
help="Enable Hybrid Sharded Data Parallel to shard model weights across GPUs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-shard-size",
|
||||
type=int,
|
||||
default=-1,
|
||||
help=(
|
||||
"Number of GPUs to shard model weights across within each replica group. "
|
||||
"-1 (default) auto-calculates as world_size / replicate_size."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-replicate-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Number of replica groups for HSDP. Each replica holds a full sharded copy. "
|
||||
"Default 1 means pure sharding (no replication)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
@@ -234,6 +257,9 @@ def main():
|
||||
vae_patch_parallel_size=args.vae_patch_parallel_size,
|
||||
ring_degree=args.ring_degree,
|
||||
ulysses_degree=args.ulysses_degree,
|
||||
use_hsdp=args.use_hsdp,
|
||||
hsdp_shard_size=args.hsdp_shard_size,
|
||||
hsdp_replicate_size=args.hsdp_replicate_size,
|
||||
)
|
||||
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
|
||||
Reference in new issue
Block a user