[Core][Frontend] Support request-level batching for diffusion pipelines (#4079)

Signed-off-by: jader <yjader@foxmail.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
JiangJie ZhangandSamit authored and GitHub committed 2026-06-29 18:09:19 +02:00
1 parent 504698dec8
commit c45ac74d43
123 files changed
+4160 -2233

No files matched your search

+102 -125
View File
@@ -29,7 +29,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def _make_request(req_id: str) -> OmniDiffusionRequest:
return OmniDiffusionRequest(
prompts=[f"prompt_{req_id}"],
prompt=f"prompt_{req_id}",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
request_id=req_id,
)
@@ -67,7 +67,7 @@ def _make_step_request(
sampling_params: OmniDiffusionSamplingParams | None = None,
) -> OmniDiffusionRequest:
return OmniDiffusionRequest(
prompts=[f"prompt_{req_id}"],
prompt=f"prompt_{req_id}",
sampling_params=sampling_params
or OmniDiffusionSamplingParams(
num_inference_steps=num_inference_steps,
@@ -128,6 +128,12 @@ class _StubScheduler(SchedulerInterface):
def has_requests(self) -> bool:
return not self._scheduled
def num_waiting_requests(self) -> int:
return 0 if self._scheduled else 1
def num_running_requests(self) -> int:
return 1 if self._scheduled else 0
def get_request_state(self, request_id: str):
del request_id
return self._state
@@ -164,7 +170,7 @@ class TestGetSamplingParamsKey:
)
sp.lora_scale = lora_scale
return OmniDiffusionRequest(
prompts=["prompt"],
prompt="prompt",
sampling_params=sp,
request_id=f"req-{lora_int_id}-{lora_scale}",
)
@@ -196,6 +202,41 @@ class TestGetSamplingParamsKey:
assert a == b
class TestGetRequestBatchSamplingParamsKey:
"""Pure-function tests for the request-batch compatibility key builder."""
@staticmethod
def _make(
*,
num_inference_steps: int = 2,
seed: int | None = 123,
generator: torch.Generator | None = None,
) -> OmniDiffusionRequest:
sp = OmniDiffusionSamplingParams(
num_inference_steps=num_inference_steps,
seed=seed,
generator=generator,
)
return OmniDiffusionRequest(prompt="prompt", sampling_params=sp, request_id=f"req-{num_inference_steps}")
def test_distinguishes_num_inference_steps(self) -> None:
from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key
assert get_request_batch_sampling_params_key(
self._make(num_inference_steps=2)
) != get_request_batch_sampling_params_key(self._make(num_inference_steps=4))
def test_ignores_seed_and_generator(self) -> None:
from vllm_omni.diffusion.sched.base_scheduler import get_request_batch_sampling_params_key
gen_a = torch.Generator(device="cpu").manual_seed(1)
gen_b = torch.Generator(device="cpu").manual_seed(2)
assert get_request_batch_sampling_params_key(
self._make(seed=1, generator=gen_a)
) == get_request_batch_sampling_params_key(self._make(seed=2, generator=gen_b))
class TestRequestScheduler:
def setup_method(self) -> None:
self.scheduler: RequestScheduler = RequestScheduler()
@@ -297,8 +338,18 @@ class TestRequestScheduler:
scheduler = RequestScheduler()
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
req_id_a = scheduler.add_request(_make_request("a"))
req_id_b = scheduler.add_request(_make_request("b"))
req_id_a = scheduler.add_request(
_make_step_request(
"a",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, seed=123),
)
)
req_id_b = scheduler.add_request(
_make_step_request(
"b",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1, seed=123),
)
)
sched_output = scheduler.schedule()
@@ -306,6 +357,50 @@ class TestRequestScheduler:
assert sched_output.num_running_reqs == 2
assert sched_output.num_waiting_reqs == 0
def test_batches_incompatible_request_sampling_params_separately(self) -> None:
scheduler = RequestScheduler()
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
req_id_a = scheduler.add_request(
_make_step_request(
"a", num_inference_steps=2, sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=123)
)
)
scheduler.add_request(
_make_step_request(
"b", num_inference_steps=4, sampling_params=OmniDiffusionSamplingParams(num_inference_steps=4, seed=123)
)
)
first = scheduler.schedule()
assert _new_ids(first) == [req_id_a]
assert first.num_running_reqs == 1
assert first.num_waiting_reqs == 1
def test_batches_different_request_local_seed_together(self) -> None:
scheduler = RequestScheduler()
scheduler.initialize(SimpleNamespace(max_num_seqs=2))
req_id_a = scheduler.add_request(
_make_step_request(
"a",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=123),
)
)
req_id_b = scheduler.add_request(
_make_step_request(
"b",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2, seed=456),
)
)
first = scheduler.schedule()
assert _new_ids(first) == [req_id_a, req_id_b]
assert first.num_running_reqs == 2
assert first.num_waiting_reqs == 0
def test_incompatible_waiting_head_blocks_later_compatible_request(self) -> None:
scheduler = RequestScheduler()
scheduler.initialize(SimpleNamespace(max_num_seqs=3))
@@ -313,7 +408,7 @@ class TestRequestScheduler:
req_id_a = scheduler.add_request(_make_request("a"))
req_id_b = scheduler.add_request(
OmniDiffusionRequest(
prompts=["prompt_b"],
prompt="prompt_b",
sampling_params=OmniDiffusionSamplingParams(width=768),
request_id="b",
)
@@ -374,7 +469,7 @@ class TestRequestScheduler:
def test_request_id_is_scheduler_key(self) -> None:
request = OmniDiffusionRequest(
prompts=["prompt_map_a", "prompt_map_b"],
prompt="prompt_map_a",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
request_id="map-parent",
)
@@ -618,124 +713,6 @@ class TestDiffusionEngine:
with pytest.raises(RuntimeError, match="Dummy run failed: boom"):
engine._dummy_run()
@pytest.mark.asyncio
async def test_step_multi_request_reuses_multimodal_slice_logic(self, mocker: MockerFixture) -> None:
engine = DiffusionEngine.__new__(DiffusionEngine)
engine.od_config = SimpleNamespace(
model_class_name="mock_model",
enable_cpu_offload=False,
)
engine.pre_process_func = None
engine.post_process_func = None
engine._check_and_start_background_loop = mocker.AsyncMock()
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
return_value=DiffusionOutput(
output={
"video": ["frame-0", "frame-1"],
"audio": ["audio-0", "audio-1"],
"actions": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
}
)
)
request = OmniDiffusionRequest(
prompts=["prompt-0", "prompt-1"],
sampling_params=OmniDiffusionSamplingParams(
num_inference_steps=1,
num_outputs_per_prompt=1,
),
request_id="req-batch",
)
mocker.patch("vllm_omni.diffusion.output_formatter.supports_audio_output", return_value=False)
outputs = await engine.step(request)
assert len(outputs) == 2
assert outputs[0].images == ["frame-0"]
assert outputs[1].images == ["frame-1"]
assert outputs[0].multimodal_output["audio"] == "audio-0"
assert outputs[1].multimodal_output["audio"] == "audio-1"
torch.testing.assert_close(
outputs[0].multimodal_output["actions"],
torch.tensor([1.0, 2.0]),
)
torch.testing.assert_close(
outputs[1].multimodal_output["actions"],
torch.tensor([3.0, 4.0]),
)
@pytest.mark.asyncio
async def test_step_empty_dict_output_still_runs_postprocess(self, mocker: MockerFixture) -> None:
engine = DiffusionEngine.__new__(DiffusionEngine)
engine.od_config = SimpleNamespace(
model_class_name="mock_model",
enable_cpu_offload=False,
)
engine.pre_process_func = None
engine.post_process_func = mocker.Mock(return_value={"video": ["processed"]})
engine._post_process_accepts_sampling_params = False
engine._check_and_start_background_loop = mocker.AsyncMock()
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
return_value=DiffusionOutput(
output={},
custom_output={"actions": torch.tensor([[1.0, 2.0]])},
)
)
request = OmniDiffusionRequest(
prompts=["prompt"],
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
request_id="req-action",
)
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
outputs = await engine.step(request)
engine.post_process_func.assert_called_once_with({})
assert outputs[0].images == ["processed"]
torch.testing.assert_close(outputs[0].multimodal_output["actions"], torch.tensor([[1.0, 2.0]]))
@pytest.mark.asyncio
async def test_step_action_only_flag_skips_postprocess(self, mocker: MockerFixture) -> None:
engine = DiffusionEngine.__new__(DiffusionEngine)
engine.od_config = SimpleNamespace(
model_class_name="mock_model",
enable_cpu_offload=False,
)
engine.pre_process_func = None
engine.post_process_func = mocker.Mock(side_effect=AssertionError("postprocess should be skipped"))
engine.action_post_process_func = mocker.Mock(return_value=torch.tensor([[3.0, 4.0]]))
engine._post_process_accepts_sampling_params = False
engine._action_post_process_accepts_custom_output = True
engine._action_post_process_accepts_sampling_params = False
engine._check_and_start_background_loop = mocker.AsyncMock()
raw_action = torch.tensor([[1.0, 2.0]])
engine.async_add_req_and_wait_for_response = mocker.AsyncMock(
return_value=DiffusionOutput(
output={},
custom_output={
"action": raw_action,
"action_only_output": True,
},
)
)
request = OmniDiffusionRequest(
prompts=["prompt"],
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
request_id="req-action",
)
mocker.patch("vllm_omni.diffusion.diffusion_engine.supports_audio_output", return_value=False)
outputs = await engine.step(request)
engine.post_process_func.assert_not_called()
engine.action_post_process_func.assert_called_once()
assert engine.action_post_process_func.call_args.args[0] is raw_action
assert "custom_output" in engine.action_post_process_func.call_args.kwargs
assert outputs[0].images == []
torch.testing.assert_close(outputs[0].multimodal_output["actions"], torch.tensor([[3.0, 4.0]]))
class TestStepScheduler:
def setup_method(self) -> None: