[Core][Frontend] Support request-level batching for diffusion pipelines (#4079)
Signed-off-by: jader <yjader@foxmail.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
1 parent
504698dec8
commit
c45ac74d43
123 files changed
+4160
-2233
No files matched your search
@@ -29,7 +29,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
def _make_request(req_id: str) -> OmniDiffusionRequest:
|
||||
return OmniDiffusionRequest(
|
||||
prompts=[f"prompt_{req_id}"],
|
||||
prompt=f"prompt_{req_id}",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
request_id=req_id,
|
||||
)
|
||||
@@ -67,7 +67,7 @@ def _make_step_request(
|
||||
sampling_params: OmniDiffusionSamplingParams | None = None,
|
||||
) -> OmniDiffusionRequest:
|
||||
return OmniDiffusionRequest(
|
||||
prompts=[f"prompt_{req_id}"],
|
||||
prompt=f"prompt_{req_id}",
|
||||
sampling_params=sampling_params
|
||||
or OmniDiffusionSamplingParams(
|
||||
num_inference_steps=num_inference_steps,
|
||||
@@ -128,6 +128,12 @@ class _StubScheduler(SchedulerInterface):
|
||||
def has_requests(self) -> bool:
|
||||
return not self._scheduled
|
||||
|
||||
def num_waiting_requests(self) -> int:
|
||||
return 0 if self._scheduled else 1
|
||||
|
||||
def num_running_requests(self) -> int:
|
||||
return 1 if self._scheduled else 0
|
||||
|
||||
def get_request_state(self, request_id: str):
|
||||
del request_id
|
||||
return self._state
|
||||
@@ -164,7 +170,7 @@ class TestGetSamplingParamsKey:
|
||||
)
|
||||
sp.lora_scale = lora_scale
|
||||
return OmniDiffusionRequest(
|
||||
prompts=["prompt"],
|
||||
prompt="prompt",
|
||||
sampling_params=sp,
|
||||
request_id=f"req-{lora_int_id}-{lora_scale}",
|
||||
)
|
||||
@@ -196,6 +202,41 @@ class TestGetSamplingParamsKey:
|
||||
assert a == b
|
||||
|
||||
|
||||
class TestGetRequestBatchSamplingParamsKey:
|
||||
"""Pure-function tests for the request-batch compatibility key builder."""
|
||||
|
||||
@staticmethod
|
||||
def _make(
|
||||
*,
|
||||
num_inference_steps: int = 2,
|
||||
seed: int | None = 123,
|
||||
generator: torch.Generator | None = None,
|
||||
) -> OmniDiffusionRequest:
|
||||
sp = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
generator=generator,
|
||||
)
|
||||
return OmniDiffusionRequest(prompt="prompt", sampling_params=sp, request_id=f"req-{num_inference_steps}")
|
||||
|
||||
def test_distinguishes_num_inference_steps(self) -> None:
|
||||
from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key
|
||||
|
||||
assert get_request_batch_sampling_params_key(
|
||||
self._make(num_inference_steps=2)
|
||||
) != get_request_batch_sampling_params_key(self._make(num_inference_steps=4))
|
||||
|
||||
def test_ignores_seed_and_generator(self) -> None:
|
||||
from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key
|
||||
|
||||
gen_a = torch.Generator(device="cpu").manual_seed(1)
|
||||
gen_b = torch.Generator(device="cpu").manual_seed(2)
|
||||
|
||||
assert get_request_batch_sampling_params_key(
|
||||
self._make(seed=1, generator=gen_a)
|
||||
) == get_request_batch_sampling_params_key(self._make(seed=2, generator=gen_b))
|
||||
|
||||
|
||||
class TestRequestScheduler:
|
||||
def setup_method(self) -> None:
|
||||
self.scheduler: RequestScheduler = RequestScheduler()
|
||||
@@ -297,8 +338,18 @@ class TestRequestScheduler:
|
||||
scheduler = RequestScheduler()
|
||||
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
|
||||
|
||||
req_id_a = scheduler.add_request(_make_request("a"))
|
||||
req_id_b = scheduler.add_request(_make_request("b"))
|
||||
req_id_a = scheduler.add_request(
|
||||
_make_step_request(
|
||||
"a",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, seed=123),
|
||||
)
|
||||
)
|
||||
req_id_b = scheduler.add_request(
|
||||
_make_step_request(
|
||||
"b",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, seed=123),
|
||||
)
|
||||
)
|
||||
|
||||
sched_output = scheduler.schedule()
|
||||
|
||||
@@ -306,6 +357,50 @@ class TestRequestScheduler:
|
||||
assert sched_output.num_running_reqs == 2
|
||||
assert sched_output.num_waiting_reqs == 0
|
||||
|
||||
def test_batches_incompatible_request_sampling_params_separately(self) -> None:
|
||||
scheduler = RequestScheduler()
|
||||
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
|
||||
|
||||
req_id_a = scheduler.add_request(
|
||||
_make_step_request(
|
||||
"a", num_inference_steps=2, sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=123)
|
||||
)
|
||||
)
|
||||
scheduler.add_request(
|
||||
_make_step_request(
|
||||
"b", num_inference_steps=4, sampling_params=OmniDiffusionSamplingParams(num_inference_steps=4, seed=123)
|
||||
)
|
||||
)
|
||||
|
||||
first = scheduler.schedule()
|
||||
|
||||
assert _new_ids(first) == [req_id_a]
|
||||
assert first.num_running_reqs == 1
|
||||
assert first.num_waiting_reqs == 1
|
||||
|
||||
def test_batches_different_request_local_seed_together(self) -> None:
|
||||
scheduler = RequestScheduler()
|
||||
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
|
||||
|
||||
req_id_a = scheduler.add_request(
|
||||
_make_step_request(
|
||||
"a",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=123),
|
||||
)
|
||||
)
|
||||
req_id_b = scheduler.add_request(
|
||||
_make_step_request(
|
||||
"b",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=456),
|
||||
)
|
||||
)
|
||||
|
||||
first = scheduler.schedule()
|
||||
|
||||
assert _new_ids(first) == [req_id_a, req_id_b]
|
||||
assert first.num_running_reqs == 2
|
||||
assert first.num_waiting_reqs == 0
|
||||
|
||||
def test_incompatible_waiting_head_blocks_later_compatible_request(self) -> None:
|
||||
scheduler = RequestScheduler()
|
||||
scheduler.initialize(SimpleNamespace(max_num_seqs=3))
|
||||
@@ -313,7 +408,7 @@ class TestRequestScheduler:
|
||||
req_id_a = scheduler.add_request(_make_request("a"))
|
||||
req_id_b = scheduler.add_request(
|
||||
OmniDiffusionRequest(
|
||||
prompts=["prompt_b"],
|
||||
prompt="prompt_b",
|
||||
sampling_params=OmniDiffusionSamplingParams(width=768),
|
||||
request_id="b",
|
||||
)
|
||||
@@ -374,7 +469,7 @@ class TestRequestScheduler:
|
||||
|
||||
def test_request_id_is_scheduler_key(self) -> None:
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=["prompt_map_a", "prompt_map_b"],
|
||||
prompt="prompt_map_a",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
request_id="map-parent",
|
||||
)
|
||||
@@ -618,124 +713,6 @@ class TestDiffusionEngine:
|
||||
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
|
||||
engine._dummy_run()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(
|
||||
model_class_name="mock_model",
|
||||
enable_cpu_offload=False,
|
||||
)
|
||||
engine.pre_process_func = None
|
||||
engine.post_process_func = None
|
||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||
return_value=DiffusionOutput(
|
||||
output={
|
||||
"video": ["frame-0", "frame-1"],
|
||||
"audio": ["audio-0", "audio-1"],
|
||||
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=["prompt-0", "prompt-1"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
num_inference_steps=1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
request_id="req-batch",
|
||||
)
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.output_formatter.supports_audio_output", return_value=False)
|
||||
outputs = await engine.step(request)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert outputs[0].images == ["frame-0"]
|
||||
assert outputs[1].images == ["frame-1"]
|
||||
assert outputs[0].multimodal_output["audio"] == "audio-0"
|
||||
assert outputs[1].multimodal_output["audio"] == "audio-1"
|
||||
torch.testing.assert_close(
|
||||
outputs[0].multimodal_output["actions"],
|
||||
torch.tensor([1.0, 2.0]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
outputs[1].multimodal_output["actions"],
|
||||
torch.tensor([3.0, 4.0]),
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_empty_dict_output_still_runs_postprocess(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(
|
||||
model_class_name="mock_model",
|
||||
enable_cpu_offload=False,
|
||||
)
|
||||
engine.pre_process_func = None
|
||||
engine.post_process_func = mocker.Mock(return_value={"video": ["processed"]})
|
||||
engine._post_process_accepts_sampling_params = False
|
||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||
return_value=DiffusionOutput(
|
||||
output={},
|
||||
custom_output={"actions": torch.tensor([[1.0, 2.0]])},
|
||||
)
|
||||
)
|
||||
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=["prompt"],
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
request_id="req-action",
|
||||
)
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
||||
outputs = await engine.step(request)
|
||||
|
||||
engine.post_process_func.assert_called_once_with({})
|
||||
assert outputs[0].images == ["processed"]
|
||||
torch.testing.assert_close(outputs[0].multimodal_output["actions"], torch.tensor([[1.0, 2.0]]))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_action_only_flag_skips_postprocess(self, mocker: MockerFixture) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(
|
||||
model_class_name="mock_model",
|
||||
enable_cpu_offload=False,
|
||||
)
|
||||
engine.pre_process_func = None
|
||||
engine.post_process_func = mocker.Mock(side_effect=AssertionError("postprocess should be skipped"))
|
||||
engine.action_post_process_func = mocker.Mock(return_value=torch.tensor([[3.0, 4.0]]))
|
||||
engine._post_process_accepts_sampling_params = False
|
||||
engine._action_post_process_accepts_custom_output = True
|
||||
engine._action_post_process_accepts_sampling_params = False
|
||||
engine._check_and_start_background_loop = mocker.AsyncMock()
|
||||
raw_action = torch.tensor([[1.0, 2.0]])
|
||||
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
|
||||
return_value=DiffusionOutput(
|
||||
output={},
|
||||
custom_output={
|
||||
"action": raw_action,
|
||||
"action_only_output": True,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
request = OmniDiffusionRequest(
|
||||
prompts=["prompt"],
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
request_id="req-action",
|
||||
)
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
|
||||
outputs = await engine.step(request)
|
||||
|
||||
engine.post_process_func.assert_not_called()
|
||||
engine.action_post_process_func.assert_called_once()
|
||||
assert engine.action_post_process_func.call_args.args[0] is raw_action
|
||||
assert "custom_output" in engine.action_post_process_func.call_args.kwargs
|
||||
assert outputs[0].images == []
|
||||
torch.testing.assert_close(outputs[0].multimodal_output["actions"], torch.tensor([[3.0, 4.0]]))
|
||||
|
||||
|
||||
class TestStepScheduler:
|
||||
def setup_method(self) -> None:
|
||||
|
||||
Reference in new issue
Block a user