[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
9ece725fc5
commit
d64bbde561
64 files changed
+2995
-1414
No files matched your search
@@ -20,7 +20,7 @@ if __name__ == "__main__":
|
||||
images[0].save("coffee.png")
|
||||
```
|
||||
|
||||
Or put more than one prompt in a request, processing them sequentially.
|
||||
Or put more than one prompt in a request.
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -37,6 +37,41 @@ if __name__ == "__main__":
|
||||
image = output.request_output[0].images[0].save(f"{i}.jpg")
|
||||
```
|
||||
|
||||
!!! info
|
||||
|
||||
However, it is not currently recommended to do so
|
||||
because not all models support batch inference,
|
||||
and batch requesting mostly does not provide significant performance improvement (despite the impression that it does).
|
||||
This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements.
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input
|
||||
list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support
|
||||
batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts.
|
||||
|
||||
Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM.
|
||||
This is useful for models that support negative prompts.
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
if __name__ == "__main__":
|
||||
omni = Omni(model="Qwen/Qwen-Image")
|
||||
outputs = omni.generate([
|
||||
{
|
||||
"prompt": "a cup of coffee on a table",
|
||||
"negative_prompt": "low resolution"
|
||||
},
|
||||
{
|
||||
"prompt": "a toy dinosaur on a sandy beach",
|
||||
"negative_prompt": "cinematic, realistic"
|
||||
}
|
||||
])
|
||||
for i, output in enumerate(outputs):
|
||||
image = output.request_output[0].images[0].save(f"{i}.jpg")
|
||||
```
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
```bash
|
||||
|
||||
@@ -5,6 +5,7 @@ import gradio as gr
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
|
||||
@@ -101,12 +102,14 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
outputs = omni.generate(
|
||||
prompt.strip(),
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
true_cfg_scale=float(cfg_scale_value),
|
||||
num_inference_steps=num_steps,
|
||||
num_outputs_per_prompt=num_images,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
true_cfg_scale=float(cfg_scale_value),
|
||||
num_inference_steps=num_steps,
|
||||
num_outputs_per_prompt=num_images,
|
||||
),
|
||||
)
|
||||
images_outputs = []
|
||||
for output in outputs:
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
|
||||
@@ -189,15 +190,19 @@ def main():
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
outputs = omni.generate(
|
||||
args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
generator=generator,
|
||||
true_cfg_scale=args.cfg_scale,
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_outputs_per_prompt=args.num_images_per_prompt,
|
||||
{
|
||||
"prompt": args.prompt,
|
||||
"negative_prompt": args.negative_prompt,
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
generator=generator,
|
||||
true_cfg_scale=args.cfg_scale,
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_outputs_per_prompt=args.num_images_per_prompt,
|
||||
),
|
||||
)
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
Reference in new issue
Block a user