[Core][Frontend] Support request-level batching for diffusion pipelines (#4079)
Signed-off-by: jader <yjader@foxmail.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
1 parent
504698dec8
commit
c45ac74d43
123 files changed
+4160
-2233
No files matched your search
@@ -114,6 +114,23 @@ class _InterruptingStepPipeline(_StepPipeline):
|
||||
raise AssertionError("post_decode should not run after interrupt")
|
||||
|
||||
|
||||
class _FakePeakMemoryPlatform:
|
||||
def __init__(self, reserved_mb: list[float]):
|
||||
self._reserved_mb = reserved_mb
|
||||
self.reset_calls = 0
|
||||
|
||||
def reset_peak_memory_stats(self):
|
||||
self.reset_calls += 1
|
||||
|
||||
def max_memory_reserved(self):
|
||||
index = min(self.reset_calls - 1, len(self._reserved_mb) - 1)
|
||||
return int(self._reserved_mb[index] * 1024**2)
|
||||
|
||||
def max_memory_allocated(self):
|
||||
index = min(self.reset_calls - 1, len(self._reserved_mb) - 1)
|
||||
return int((self._reserved_mb[index] - 100) * 1024**2)
|
||||
|
||||
|
||||
class _IdentityNoiseTransformer(torch.nn.Module):
|
||||
def forward(self, x: torch.Tensor, **kwargs):
|
||||
del kwargs
|
||||
@@ -205,15 +222,10 @@ class _DistributedStepPipeline(CFGParallelMixin):
|
||||
|
||||
|
||||
def _make_step_request(num_inference_steps: int = 2):
|
||||
return SimpleNamespace(
|
||||
prompts=["a prompt"],
|
||||
return OmniDiffusionRequest(
|
||||
prompt="a prompt",
|
||||
request_id="req-1",
|
||||
sampling_params=SimpleNamespace(
|
||||
generator=None,
|
||||
seed=None,
|
||||
generator_device=None,
|
||||
num_inference_steps=num_inference_steps,
|
||||
),
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=num_inference_steps),
|
||||
)
|
||||
|
||||
|
||||
@@ -226,7 +238,7 @@ def _assert_aborted_output(output: DiffusionOutput, request_id: str) -> None:
|
||||
|
||||
def _make_engine_request(req_id: str = "req-1", num_inference_steps: int = 2) -> OmniDiffusionRequest:
|
||||
return OmniDiffusionRequest(
|
||||
prompts=[f"prompt-{req_id}"],
|
||||
prompt=f"prompt-{req_id}",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=num_inference_steps),
|
||||
request_id=req_id,
|
||||
)
|
||||
@@ -282,6 +294,7 @@ def _make_distributed_runner(mode: str, device: torch.device):
|
||||
|
||||
|
||||
def _make_scheduler_output(req, request_id="req-1", step_id=0, finished_req_ids=None):
|
||||
req.request_id = request_id
|
||||
return DiffusionSchedulerOutput(
|
||||
step_id=step_id,
|
||||
scheduled_new_reqs=[NewRequestData(request_id=request_id, req=req)],
|
||||
@@ -309,7 +322,7 @@ def _make_input_batch_state(request_id: str, latent_value: float) -> DiffusionRe
|
||||
state = DiffusionRequestState(
|
||||
request_id=request_id,
|
||||
sampling=SimpleNamespace(),
|
||||
prompts=None,
|
||||
prompt=None,
|
||||
)
|
||||
state.latents = torch.tensor([[latent_value]])
|
||||
state.timesteps = torch.tensor([1.0])
|
||||
@@ -456,6 +469,24 @@ class TestRunner:
|
||||
assert runner.pipeline.scheduler_calls == 2
|
||||
assert runner.pipeline.decode_calls == 1
|
||||
|
||||
def test_carries_peak_memory_across_stepwise_request_lifecycle(self, monkeypatch):
|
||||
runner = _make_runner()
|
||||
req = _make_step_request()
|
||||
fake_platform = _FakePeakMemoryPlatform([1500.0, 1200.0])
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(model_runner_module, "current_omni_platform", fake_platform)
|
||||
|
||||
first = DiffusionModelRunner.execute_stepwise(runner, _make_scheduler_output(req, step_id=0))
|
||||
first_output = first.get_request_output("req-1")
|
||||
assert first_output.finished is False
|
||||
assert first_output.result is None
|
||||
|
||||
second = DiffusionModelRunner.execute_stepwise(runner, _make_cached_scheduler_output(step_id=1))
|
||||
second_output = second.get_request_output("req-1")
|
||||
assert second_output.finished is True
|
||||
assert second_output.result is not None
|
||||
assert second_output.result.peak_memory_mb == pytest.approx(1500.0)
|
||||
|
||||
def test_rejects_multi_request_step_batch(self):
|
||||
runner = _make_runner()
|
||||
req_1 = _make_step_request()
|
||||
|
||||
Reference in new issue
Block a user