[Feat] support for multi-block layerwise offloading, fix top-level parameters/buffers staying on CPU (#1486)
Signed-off-by: Lancer <maruixiang6688@gmail.com> Signed-off-by: Lancer <402430575@qq.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
e23b2634d1
commit
328de586aa
13 files changed
+288
-38
No files matched your search
@@ -1,6 +1,12 @@
|
||||
"""
|
||||
Tests for Flux2 Klein; currently Dev is implemented separately,
|
||||
but ideally these models will fold together in the future.
|
||||
|
||||
Coverage:
|
||||
- FP8 + CacheDiT + Ulysses=2 + TP=2
|
||||
- Layerwise CPU offload + Ulysses=2 + Ring=2
|
||||
- Layerwise CPU offload + TP=2
|
||||
- Layerwise CPU offload + HSDP
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -42,6 +48,45 @@ def _get_diffusion_feature_cases(model: str):
|
||||
),
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--ulysses-degree",
|
||||
"2",
|
||||
"--ring",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="layerwise_ulysses2_ring2",
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="layerwise_tp2",
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--use-hsdp",
|
||||
"--hsdp-shard-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="layerwise_hsdp",
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -3,9 +3,12 @@ Tests of common diffusion feature combinations in online serving mode
|
||||
for Z-Image.
|
||||
|
||||
Coverage is intentionally limited to the minimal 4xL4 cases that
|
||||
exercise Z-Image's supported parallel feature combinations:
|
||||
exercise Z-Image's supported feature combinations:
|
||||
- CacheDiT + FP8 + Ring=2 + TP=2
|
||||
- TeaCache + FP8 + Ulysses=2 + Ring=2
|
||||
- Layerwise CPU offload + Ulysses=2 + Ring=2
|
||||
- Layerwise CPU offload + TP=2
|
||||
- Layerwise CPU offload + HSDP
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -64,12 +67,39 @@ def _get_diffusion_feature_cases():
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--ulysses-degree",
|
||||
"2",
|
||||
"--ring",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="layerwise_ulysses2_ring2",
|
||||
marks=FOUR_CARD_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="layerwise_tp2",
|
||||
marks=FOUR_CARD_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
server_args=[
|
||||
"--enable-layerwise-offload",
|
||||
"--use-hsdp",
|
||||
"--hsdp-shard-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_hsdp",
|
||||
id="layerwise_hsdp",
|
||||
marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")],
|
||||
),
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user