diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 648c00ffd..1b6334d3f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -127,3 +127,38 @@ steps: hostPath: path: /mnt/hf-cache type: DirectoryOrCreate + + - label: "Diffusion Image Edit Test with H100 (1 GPU)" + timeout_in_minutes: 20 + depends_on: image-build + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate diff --git a/docs/user_guide/acceleration/cache_dit_acceleration.md b/docs/user_guide/acceleration/cache_dit_acceleration.md index 35aeae3c9..10ffe321e 100644 --- a/docs/user_guide/acceleration/cache_dit_acceleration.md +++ b/docs/user_guide/acceleration/cache_dit_acceleration.md @@ -50,6 +50,22 @@ omni = Omni( ) ``` +## Online Serving (OpenAI-Compatible) + +Enable Cache-DiT for online serving by passing `--cache-backend cache_dit` when starting the server: + +```bash +# Use Cache-DiT default (recommended) parameters +vllm serve Qwen/Qwen-Image --omni --port 8091 --cache-backend cache_dit +``` + +To customize Cache-DiT settings for online serving, pass a JSON string via `--cache-config`: + +```bash +vllm serve Qwen/Qwen-Image --omni --port 8091 \ + --cache-backend cache_dit \ + --cache-config '{"Fn_compute_blocks": 1, "Bn_compute_blocks": 0, "max_warmup_steps": 4, "residual_diff_threshold": 0.12}' +``` ## Acceleration Methods diff --git a/docs/user_guide/acceleration/teacache.md b/docs/user_guide/acceleration/teacache.md index ef7af50c3..ab8958929 100644 --- a/docs/user_guide/acceleration/teacache.md +++ b/docs/user_guide/acceleration/teacache.md @@ -39,6 +39,16 @@ omni = Omni( ) ``` +## Online Serving (OpenAI-Compatible) + +Enable TeaCache for online serving by passing `--cache-backend tea_cache` when starting the server: + +```bash +vllm serve Qwen/Qwen-Image --omni --port 8091 \ + --cache-backend tea_cache \ + --cache-config '{"rel_l1_thresh": 0.2}' +``` + ## Configuration Parameters ### `rel_l1_thresh` (float, default: `0.2`) diff --git a/docs/user_guide/diffusion_acceleration.md b/docs/user_guide/diffusion_acceleration.md index 819b0918e..64b874b42 100644 --- a/docs/user_guide/diffusion_acceleration.md +++ b/docs/user_guide/diffusion_acceleration.md @@ -36,7 +36,7 @@ The following table shows which models are currently supported by each accelerat |-------|-----------------|----------|-----------|-----------| | **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | | **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ✅ |❌ | -| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ❌ | ✅ |✅ | +| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ |✅ | | **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ❌ | ✅ |✅ | diff --git a/docs/user_guide/examples/offline_inference/image_to_image.md b/docs/user_guide/examples/offline_inference/image_to_image.md index b0373e002..700b2e525 100644 --- a/docs/user_guide/examples/offline_inference/image_to_image.md +++ b/docs/user_guide/examples/offline_inference/image_to_image.md @@ -19,7 +19,7 @@ python image_edit.py \ --guidance_scale 1.0 ``` -### Multiple Image Editing (Qwen-Image-Edit-2509+) +### Multiple Image Editing (Qwen-Image-Edit-2509) For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or later version: diff --git a/examples/offline_inference/image_to_image/image_to_image.md b/examples/offline_inference/image_to_image/image_to_image.md index b2c40dd85..2e1662085 100644 --- a/examples/offline_inference/image_to_image/image_to_image.md +++ b/examples/offline_inference/image_to_image/image_to_image.md @@ -15,7 +15,7 @@ python image_edit.py \ --cfg_scale 4.0 \ ``` -### Multiple Image Editing (Qwen-Image-Edit-2509+) +### Multiple Image Editing (Qwen-Image-Edit-2509) For multiple image inputs, use `Qwen/Qwen-Image-Edit-2509` or later version: diff --git a/examples/online_serving/image_to_image/README.md b/examples/online_serving/image_to_image/README.md index 53f0faa30..097907c42 100644 --- a/examples/online_serving/image_to_image/README.md +++ b/examples/online_serving/image_to_image/README.md @@ -2,6 +2,8 @@ This example demonstrates how to deploy Qwen-Image-Edit model for online image editing service using vLLM-Omni. +For **multi-image** input editing, use **Qwen-Image-Edit-2509** (QwenImageEditPlusPipeline) and send multiple images in the user message content. + ## Start Server ### Basic Start @@ -10,6 +12,12 @@ This example demonstrates how to deploy Qwen-Image-Edit model for online image e vllm serve Qwen/Qwen-Image-Edit --omni --port 8092 ``` +### Multi-Image Edit (Qwen-Image-Edit-2509) + +```bash +vllm serve Qwen/Qwen-Image-Edit-2509 --omni --port 8092 +``` + ### Start with Parameters @@ -19,6 +27,12 @@ Or use the startup script: bash run_server.sh ``` +To serve Qwen-Image-Edit-2509 with the script: + +```bash +MODEL=Qwen/Qwen-Image-Edit-2509 bash run_server.sh +``` + ## API Calls ### Method 1: Using curl (Image Editing) @@ -53,6 +67,9 @@ curl -s http://localhost:8092/v1/chat/completions \ ```bash python openai_chat_client.py --input input.png --prompt "Convert to oil painting style" --output output.png + +# Multi-image editing (Qwen-Image-Edit-2509 server required) +python openai_chat_client.py --input input1.png input2.png --prompt "Combine these images into a single scene" --output output.png ``` ### Method 3: Using Gradio Demo @@ -121,6 +138,25 @@ Use `extra_body` to pass generation parameters: } ``` +### Multi-Image Editing (Qwen-Image-Edit-2509) + +Provide multiple images in `content` (order matters): + +```json +{ + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Combine these images into a single scene"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."} }, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."} } + ] + } + ] +} +``` + ## Generation Parameters (extra_body) | Parameter | Type | Default | Description | diff --git a/examples/online_serving/image_to_image/gradio_demo.py b/examples/online_serving/image_to_image/gradio_demo.py index db3198c79..8cad48279 100644 --- a/examples/online_serving/image_to_image/gradio_demo.py +++ b/examples/online_serving/image_to_image/gradio_demo.py @@ -15,8 +15,15 @@ import requests from PIL import Image +def _pil_to_b64_png(img: Image.Image) -> str: + buffer = BytesIO() + img.save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode("utf-8") + + def edit_image( input_image: Image.Image, + extra_images: list[str] | None, prompt: str, steps: int, guidance_scale: float, @@ -28,19 +35,23 @@ def edit_image( if input_image is None: raise gr.Error("Please upload an image first") - # Convert image to base64 - buffer = BytesIO() - input_image.save(buffer, format="PNG") - image_b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + images: list[Image.Image] = [input_image] + if extra_images: + for p in extra_images: + try: + images.append(Image.open(p).convert("RGB")) + except Exception as e: + raise gr.Error(f"Failed to open image: {p}. Error: {e}") from e # Build user message with text and image + content: list[dict[str, object]] = [{"type": "text", "text": prompt}] + for img in images: + content.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{_pil_to_b64_png(img)}"}}) + messages = [ { "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}}, - ], + "content": content, } ] @@ -87,7 +98,10 @@ def create_demo(server_url: str): with gr.Blocks(title="Qwen-Image-Edit Demo") as demo: gr.Markdown("# Qwen-Image-Edit Online Editing") - gr.Markdown("Upload an image and describe the editing effect you want") + gr.Markdown( + "Upload an image and describe the editing effect you want. " + "For multi-image editing, upload extra images (requires Qwen-Image-Edit-2509 server)." + ) with gr.Row(): with gr.Column(scale=1): @@ -95,6 +109,11 @@ def create_demo(server_url: str): label="Input Image", type="pil", ) + extra_images = gr.File( + label="Additional Images (Optional)", + file_count="multiple", + type="filepath", + ) prompt = gr.Textbox( label="Edit Instruction", placeholder="Describe the editing effect you want...", @@ -152,13 +171,13 @@ def create_demo(server_url: str): inputs=[prompt], ) - def process_edit(img, p, st, g, se, n): + def process_edit(img, imgs, p, st, g, se, n): actual_seed = se if se >= 0 else None - return edit_image(img, p, st, g, actual_seed, n, server_url) + return edit_image(img, imgs, p, st, g, actual_seed, n, server_url) edit_btn.click( fn=process_edit, - inputs=[input_image, prompt, steps, guidance_scale, seed, negative_prompt], + inputs=[input_image, extra_images, prompt, steps, guidance_scale, seed, negative_prompt], outputs=[output_image], ) diff --git a/examples/online_serving/image_to_image/openai_chat_client.py b/examples/online_serving/image_to_image/openai_chat_client.py index 01ec3cb1d..14bec8a3b 100644 --- a/examples/online_serving/image_to_image/openai_chat_client.py +++ b/examples/online_serving/image_to_image/openai_chat_client.py @@ -5,6 +5,7 @@ Qwen-Image-Edit OpenAI-compatible chat client for image editing. Usage: python openai_chat_client.py --input qwen_image_output.png --prompt "Convert to watercolor style" --output output.png python openai_chat_client.py --input input.png --prompt "Convert to oil painting" --seed 42 + python openai_chat_client.py --input input1.png input2.png --prompt "Combine these images into a single scene" """ import argparse @@ -16,8 +17,19 @@ import requests from PIL import Image +def _encode_image_as_data_url(input_path: Path) -> str: + image_bytes = input_path.read_bytes() + try: + img = Image.open(BytesIO(image_bytes)) + mime_type = f"image/{img.format.lower()}" if img.format else "image/png" + except Exception: + mime_type = "image/png" + image_b64 = base64.b64encode(image_bytes).decode("utf-8") + return f"data:{mime_type};base64,{image_b64}" + + def edit_image( - input_image: str | Path, + input_image: str | Path | list[str | Path], prompt: str, server_url: str = "http://localhost:8092", height: int | None = None, @@ -30,7 +42,7 @@ def edit_image( """Edit an image using the chat completions API. Args: - input_image: Path to input image + input_image: Path(s) to input image(s). For multi-image editing, pass multiple paths. prompt: Text description of the edit server_url: Server URL height: Output image height in pixels @@ -43,28 +55,22 @@ def edit_image( Returns: Edited image bytes or None if failed """ - # Read and encode input image - input_path = Path(input_image) - if not input_path.exists(): - print(f"Error: Input image not found: {input_path}") - return None - - with open(input_path, "rb") as f: - image_bytes = f.read() - image_b64 = base64.b64encode(image_bytes).decode("utf-8") - - # Detect image type - img = Image.open(BytesIO(image_bytes)) - mime_type = f"image/{img.format.lower()}" if img.format else "image/png" + input_images = input_image if isinstance(input_image, list) else [input_image] + input_paths = [Path(p) for p in input_images] + for p in input_paths: + if not p.exists(): + print(f"Error: Input image not found: {p}") + return None # Build user message with text and image + content: list[dict[str, object]] = [{"type": "text", "text": prompt}] + for p in input_paths: + content.append({"type": "image_url", "image_url": {"url": _encode_image_as_data_url(p)}}) + messages = [ { "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_b64}"}}, - ], + "content": content, } ] @@ -113,7 +119,7 @@ def edit_image( def main(): parser = argparse.ArgumentParser(description="Qwen-Image-Edit chat client") - parser.add_argument("--input", "-i", required=True, help="Input image path") + parser.add_argument("--input", "-i", required=True, nargs="+", help="Input image path(s)") parser.add_argument("--prompt", "-p", required=True, help="Edit prompt") parser.add_argument("--output", "-o", default="output.png", help="Output file") parser.add_argument("--server", "-s", default="http://localhost:8092", help="Server URL") @@ -126,7 +132,10 @@ def main(): args = parser.parse_args() - print(f"Input: {args.input}") + if len(args.input) == 1: + print(f"Input: {args.input[0]}") + else: + print(f"Inputs ({len(args.input)}): {', '.join(args.input)}") print(f"Prompt: {args.prompt}") image_bytes = edit_image( diff --git a/tests/e2e/online_serving/test_i2i_multi_image_input.py b/tests/e2e/online_serving/test_i2i_multi_image_input.py new file mode 100644 index 000000000..0d5c9a5c5 --- /dev/null +++ b/tests/e2e/online_serving/test_i2i_multi_image_input.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E online serving test for Qwen-Image-Edit-2509 multi-image input. +""" + +import base64 +import os +import socket +import subprocess +import sys +import time +from io import BytesIO + +import openai +import pytest +from PIL import Image +from vllm.assets.image import ImageAsset +from vllm.utils import get_open_port + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +models = ["Qwen/Qwen-Image-Edit-2509"] +test_params = models + + +class OmniServer: + """Omniserver for vLLM-Omni tests.""" + + def __init__( + self, + model: str, + serve_args: list[str], + *, + env_dict: dict[str, str] | None = None, + ) -> None: + self.model = model + self.serve_args = serve_args + self.env_dict = env_dict + self.proc: subprocess.Popen | None = None + self.host = "127.0.0.1" + self.port = get_open_port() + + def _start_server(self) -> None: + """Start the vLLM-Omni server subprocess.""" + env = os.environ.copy() + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + if self.env_dict is not None: + env.update(self.env_dict) + + cmd = [ + sys.executable, + "-m", + "vllm_omni.entrypoints.cli.main", + "serve", + self.model, + "--omni", + "--host", + self.host, + "--port", + str(self.port), + ] + self.serve_args + + print(f"Launching OmniServer with: {' '.join(cmd)}") + self.proc = subprocess.Popen( + cmd, + env=env, + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # Set working directory to vllm-omni root + ) + + # Wait for server to be ready + max_wait = 600 # 10 minutes + start_time = time.time() + while time.time() - start_time < max_wait: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + result = sock.connect_ex((self.host, self.port)) + if result == 0: + print(f"Server ready on {self.host}:{self.port}") + return + except Exception: + pass + time.sleep(2) + + raise RuntimeError(f"Server failed to start within {max_wait} seconds") + + def __enter__(self): + self._start_server() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.proc: + self.proc.terminate() + try: + self.proc.wait(timeout=30) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + +@pytest.fixture +def omni_server(request): + """Start vLLM-Omni server as a subprocess with actual model weights.""" + model = request.param + with OmniServer(model, ["--num-gpus", "1"]) as server: + yield server + + +@pytest.fixture +def client(omni_server): + """OpenAI client for the running vLLM-Omni server.""" + return openai.OpenAI( + base_url=f"http://{omni_server.host}:{omni_server.port}/v1", + api_key="EMPTY", + ) + + +@pytest.fixture(scope="session") +def base64_encoded_images() -> list[str]: + """Base64 encoded PNG images for testing.""" + images = [ + ImageAsset("cherry_blossom").pil_image.convert("RGB"), + ImageAsset("stop_sign").pil_image.convert("RGB"), + ] + encoded: list[str] = [] + for img in images: + with BytesIO() as buffer: + img.save(buffer, format="PNG") + encoded.append(base64.b64encode(buffer.getvalue()).decode("utf-8")) + return encoded + + +def dummy_messages_from_image_data( + image_data_urls: list[str], + content_text: str = "Combine these two images into one scene.", +): + """Create messages with image data URLs for OpenAI API.""" + content = [{"type": "text", "text": content_text}] + for image_url in image_data_urls: + content.append({"type": "image_url", "image_url": {"url": image_url}}) + return [{"role": "user", "content": content}] + + +def _extract_image_data_url(message_content) -> str: + assert isinstance(message_content, list) and len(message_content) >= 1 + content_part = message_content[0] + if isinstance(content_part, dict): + image_url = content_part.get("image_url", {}).get("url", "") + else: + image_url_obj = getattr(content_part, "image_url", None) + if isinstance(image_url_obj, dict): + image_url = image_url_obj.get("url", "") + else: + image_url = getattr(image_url_obj, "url", "") + assert isinstance(image_url, str) and image_url + return image_url + + +def _decode_data_url_to_image_bytes(data_url: str) -> bytes: + assert data_url.startswith("data:image") + _, b64_data = data_url.split(",", 1) + return base64.b64decode(b64_data) + + +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_i2i_multi_image_input_qwen_image_edit_2509( + client: openai.OpenAI, + omni_server, + base64_encoded_images: list[str], +) -> None: + """Test multi-image input editing via OpenAI API.""" + image_data_urls = [f"data:image/png;base64,{img}" for img in base64_encoded_images] + messages = dummy_messages_from_image_data(image_data_urls) + + height = 832 + width = 1248 + chat_completion = client.chat.completions.create( + model=omni_server.model, + messages=messages, + extra_body={ + "height": height, + "width": width, + "num_inference_steps": 2, + "guidance_scale": 0.0, + "seed": 42, + }, + ) + + assert len(chat_completion.choices) == 1 + choice = chat_completion.choices[0] + assert choice.finish_reason == "stop" + assert choice.message.role == "assistant" + + image_data_url = _extract_image_data_url(choice.message.content) + image_bytes = _decode_data_url_to_image_bytes(image_data_url) + img = Image.open(BytesIO(image_bytes)) + img.load() + assert img.size == (width, height) diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 34692f5c4..664ca09cb 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -335,6 +335,9 @@ class OmniDiffusionConfig: # Scheduler flow_shift for Wan2.2 (12.0 for 480p, 5.0 for 720p) flow_shift: float | None = None + # support multi images input + supports_multimodal_inputs: bool = False + # Logging log_level: str = "info" @@ -415,6 +418,9 @@ class OmniDiffusionConfig: # If it's neither dict nor DiffusionCacheConfig, convert to empty config self.cache_config = DiffusionCacheConfig() + def update_multimodal_support(self) -> None: + self.supports_multimodal_inputs = self.model_class_name in {"QwenImageEditPlusPipeline"} + @classmethod def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig": # Check environment variable as fallback for cache_backend diff --git a/vllm_omni/entrypoints/async_diffusion.py b/vllm_omni/entrypoints/async_diffusion.py index b83f624ad..079432eda 100644 --- a/vllm_omni/entrypoints/async_diffusion.py +++ b/vllm_omni/entrypoints/async_diffusion.py @@ -68,6 +68,7 @@ class AsyncOmniDiffusion: # Load model class name and transformer config config_dict = get_hf_file_to_dict("model_index.json", od_config.model) od_config.model_class_name = config_dict.get("_class_name", None) + od_config.update_multimodal_support() tf_config_dict = get_hf_file_to_dict("transformer/config.json", od_config.model) od_config.tf_model_config = TransformerConfig.from_dict(tf_config_dict) diff --git a/vllm_omni/entrypoints/omni_diffusion.py b/vllm_omni/entrypoints/omni_diffusion.py index 9f4382222..9f149f37b 100644 --- a/vllm_omni/entrypoints/omni_diffusion.py +++ b/vllm_omni/entrypoints/omni_diffusion.py @@ -53,6 +53,7 @@ class OmniDiffusion: od_config.model, ) od_config.model_class_name = config_dict.get("_class_name", None) + od_config.update_multimodal_support() tf_config_dict = get_hf_file_to_dict( "transformer/config.json", od_config.model, diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index f079be4e2..ebf947a88 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -1033,7 +1033,20 @@ class OmniOpenAIServingChat(OpenAIServingChat): # Add reference image if provided if pil_images: - gen_kwargs["pil_image"] = pil_images[0] + if len(pil_images) == 1: + gen_kwargs["pil_image"] = pil_images[0] + else: + od_config = getattr(self._diffusion_engine, "od_config", None) + supports_multimodal_inputs = getattr(od_config, "supports_multimodal_inputs", False) + if supports_multimodal_inputs: + gen_kwargs["pil_image"] = pil_images + else: + return self._create_error_response( + "Multiple input images are not supported by the current diffusion model. " + "For multi-image editing, start the server with Qwen-Image-Edit-2509 " + "and send multiple images in the user message content.", + status_code=400, + ) # Generate image result = await self._diffusion_engine.generate(**gen_kwargs)