[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)
Signed-off-by: zhuhh97 <zhuhonghan@huawei.com> Signed-off-by: dph97 <nenbaying@163.com> Signed-off-by: zhuhh97 <zhh@unknown> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
33 files changed
+1181
-135
No files matched your search
@@ -933,6 +933,7 @@ class AsyncOmniEngine:
|
||||
if parallel_config is None:
|
||||
ulysses_degree = normalized_kwargs.get("ulysses_degree") or 1
|
||||
ring_degree = normalized_kwargs.get("ring_degree") or 1
|
||||
allgather_degree = normalized_kwargs.get("allgather_degree") or 1
|
||||
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
|
||||
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
|
||||
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
|
||||
@@ -947,7 +948,7 @@ class AsyncOmniEngine:
|
||||
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
|
||||
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
|
||||
if sequence_parallel_size is None:
|
||||
sequence_parallel_size = ulysses_degree * ring_degree
|
||||
sequence_parallel_size = allgather_degree if allgather_degree > 1 else ulysses_degree * ring_degree
|
||||
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
@@ -957,6 +958,7 @@ class AsyncOmniEngine:
|
||||
sequence_parallel_size=sequence_parallel_size,
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
allgather_degree=allgather_degree,
|
||||
ulysses_mode=ulysses_mode,
|
||||
cfg_parallel_size=cfg_parallel_size,
|
||||
vae_patch_parallel_size=vae_patch_parallel_size,
|
||||
|
||||
Reference in new issue
Block a user