[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)
Signed-off-by: zhuhh97 <zhuhonghan@huawei.com> Signed-off-by: dph97 <nenbaying@163.com> Signed-off-by: zhuhh97 <zhh@unknown> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
33 files changed
+1181
-135
No files matched your search
@@ -336,6 +336,31 @@ class TestStageConfig:
|
||||
assert "ring_degree" not in omega_config.engine_args
|
||||
assert "sequence_parallel_size" not in omega_config.engine_args
|
||||
|
||||
def test_to_omegaconf_diffusion_parallel_allgather_override_recomputes_sequence_parallel_size(self):
|
||||
config = StageConfig(
|
||||
stage_id=1,
|
||||
model_stage="diffusion",
|
||||
stage_type=StageType.DIFFUSION,
|
||||
yaml_engine_args={
|
||||
"parallel_config": {
|
||||
"sequence_parallel_size": 1,
|
||||
"ulysses_degree": 1,
|
||||
"ring_degree": 1,
|
||||
"allgather_degree": 1,
|
||||
}
|
||||
},
|
||||
runtime_overrides={
|
||||
"allgather_degree": 2,
|
||||
},
|
||||
)
|
||||
|
||||
omega_config = config.to_omegaconf()
|
||||
|
||||
assert omega_config.engine_args.parallel_config.allgather_degree == 2
|
||||
assert omega_config.engine_args.parallel_config.sequence_parallel_size == 2
|
||||
assert "allgather_degree" not in omega_config.engine_args
|
||||
assert "sequence_parallel_size" not in omega_config.engine_args
|
||||
|
||||
def test_to_omegaconf_diffusion_parallel_explicit_sequence_parallel_size_is_preserved(self):
|
||||
config = StageConfig(
|
||||
stage_id=1,
|
||||
|
||||
Reference in new issue
Block a user