From 284575ab8231812685afbd1458568a3dd7aba204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Sun, 15 Mar 2026 18:27:07 +0800 Subject: [PATCH] [CI]: Add/Fix bagel e2e online/offline test (#1895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: princepride Signed-off-by: 汪志鹏 --- .buildkite/test-ready.yml | 150 ++++++++--- .../stage_configs/bagel_sharedmemory_ci.yaml | 2 +- .../offline_inference/test_bagel_img2img.py | 184 +++++++++++++ .../offline_inference/test_bagel_text2img.py | 26 +- tests/e2e/online_serving/test_bagel_online.py | 247 ++++++++++++++++++ 5 files changed, 557 insertions(+), 52 deletions(-) create mode 100644 tests/e2e/offline_inference/test_bagel_img2img.py create mode 100644 tests/e2e/online_serving/test_bagel_online.py diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index 66598f3ea..c8579979c 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -261,39 +261,117 @@ steps: # path: /mnt/hf-cache # type: DirectoryOrCreate - # - label: "Bagel Text2Img Model Test with H100" - # depends_on: upload-ready-pipeline - # commands: - # - | - # timeout 30m bash -c ' - # export VLLM_WORKER_MULTIPROC_METHOD=spawn - # pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py - # ' - # agents: - # queue: "mithril-h100-pool" - # plugins: - # - kubernetes: - # podSpec: - # containers: - # - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - # resources: - # limits: - # nvidia.com/gpu: 1 - # volumeMounts: - # - name: devshm - # mountPath: /dev/shm - # - name: hf-cache - # mountPath: /root/.cache/huggingface - # env: - # - name: HF_HOME - # value: /root/.cache/huggingface - # nodeSelector: - # node.kubernetes.io/instance-type: gpu-h100-sxm - # volumes: - # - name: devshm - # emptyDir: - # medium: Memory - # - name: hf-cache - # hostPath: - # path: /mnt/hf-cache - # type: DirectoryOrCreate + - label: "Bagel Text2Img Model Test with H100" + depends_on: upload-ready-pipeline + commands: + - | + timeout 30m bash -c ' + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_TEST_CLEAN_GPU_MEMORY=1 + pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py + ' + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Bagel Img2Img Model Test with H100" + depends_on: upload-ready-pipeline + commands: + - | + timeout 30m bash -c ' + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_TEST_CLEAN_GPU_MEMORY=1 + pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py + ' + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Bagel Online Serving Test with H100" + depends_on: upload-ready-pipeline + commands: + - | + timeout 40m bash -c ' + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export VLLM_TEST_CLEAN_GPU_MEMORY=1 + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_bagel_online.py + ' + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate diff --git a/tests/e2e/offline_inference/stage_configs/bagel_sharedmemory_ci.yaml b/tests/e2e/offline_inference/stage_configs/bagel_sharedmemory_ci.yaml index 686600f72..721c50248 100644 --- a/tests/e2e/offline_inference/stage_configs/bagel_sharedmemory_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/bagel_sharedmemory_ci.yaml @@ -10,7 +10,7 @@ stage_args: max_batch_size: 1 engine_args: model_stage: thinker - model_arch: BagelForConditionalGeneration + model_arch: OmniBagelForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.45 diff --git a/tests/e2e/offline_inference/test_bagel_img2img.py b/tests/e2e/offline_inference/test_bagel_img2img.py new file mode 100644 index 000000000..eef0b7d6c --- /dev/null +++ b/tests/e2e/offline_inference/test_bagel_img2img.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +End-to-end test for Bagel img2img generation. + +This test validates that the Bagel model generates images from an input image +and text prompt that match expected reference pixel values within a ±5 tolerance. + +Equivalent to running: + python3 examples/offline_inference/bagel/end2end.py \ + --prompts "Change the grass color to red" \ + --modality img2img --step 15 \ + --image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg +""" + +import socket +from pathlib import Path +from typing import Any + +import pytest +from PIL import Image +from vllm.assets.image import ImageAsset + +from tests.utils import hardware_test +from vllm_omni.entrypoints.omni import Omni + +# Reference pixel data extracted from the known-good output image +# Generated with seed=52, num_inference_steps=15, +# prompt='Change the grass color to red', +# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg +REFERENCE_PIXELS = [ + {"position": (100, 100), "rgb": (157, 172, 217)}, + {"position": (400, 50), "rgb": (105, 144, 218)}, + {"position": (700, 100), "rgb": (118, 159, 233)}, + {"position": (150, 400), "rgb": (195, 34, 60)}, + {"position": (512, 336), "rgb": (222, 214, 193)}, + {"position": (700, 400), "rgb": (197, 15, 43)}, + {"position": (100, 600), "rgb": (105, 13, 18)}, + {"position": (400, 600), "rgb": (169, 33, 44)}, + {"position": (700, 600), "rgb": (101, 86, 93)}, + {"position": (256, 256), "rgb": (181, 202, 222)}, +] + +PIXEL_TOLERANCE = 5 + +DEFAULT_PROMPT = "<|fim_middle|><|im_start|>Change the grass color to red<|im_end|>" + +EXPECTED_OUTPUT_SIZE = (1024, 672) + + +def _load_input_image() -> Image.Image: + """Load the test input image via vllm's ImageAsset.""" + return ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB") + + +def _find_free_port() -> int: + """Find and return a free ephemeral port by binding to port 0.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + s.listen(1) + port = s.getsockname()[1] + return port + + +def _configure_sampling_params(omni: Omni, max_tokens: int = 1, num_inference_steps: int = 15) -> list: + """Configure sampling parameters for Bagel img2img generation. + + Args: + omni: The Omni instance to get default params from. + max_tokens: Maximum tokens for the first stage. + num_inference_steps: Number of inference steps for the diffusion stage. + + Returns: + Configured sampling params list. + """ + params_list = omni.default_sampling_params_list + params_list[0].max_tokens = max_tokens # type: ignore + if len(params_list) > 1: + params_list[1].num_inference_steps = num_inference_steps # type: ignore + params_list[1].extra_args = { # type: ignore + "cfg_text_scale": 4.0, + "cfg_img_scale": 1.5, + } + return params_list + + +def _extract_generated_image(omni_outputs: list) -> Image.Image | None: + """Extract the generated image from Omni outputs. + + Args: + omni_outputs: List of outputs from omni.generate(). + + Returns: + The first generated PIL Image, or None if no image found. + """ + for req_output in omni_outputs: + if images := getattr(req_output, "images", None): + return images[0] + if hasattr(req_output, "request_output") and req_output.request_output: + for stage_out in req_output.request_output: + if hasattr(stage_out, "images") and stage_out.images: + return stage_out.images[0] + return None + + +def _validate_pixels( + image: Image.Image, + reference_pixels: list[dict[str, Any]] = REFERENCE_PIXELS, + tolerance: int = PIXEL_TOLERANCE, +) -> None: + """Validate that image pixels match expected reference values. + + Args: + image: The PIL Image to validate. + reference_pixels: List of dicts with 'position' (x, y) and 'rgb' (R, G, B). + tolerance: Maximum allowed difference per color channel. + + Raises: + AssertionError: If any pixel differs beyond tolerance. + """ + for ref in reference_pixels: + x, y = ref["position"] + expected = ref["rgb"] + actual = image.getpixel((x, y))[:3] + assert all(abs(a - e) <= tolerance for a, e in zip(actual, expected)), ( + f"Pixel mismatch at ({x}, {y}): expected {expected}, got {actual}" + ) + + +def _generate_bagel_img2img( + omni: Omni, + input_image: Image.Image, + prompt: str = DEFAULT_PROMPT, +) -> Image.Image: + """Generate an image using Bagel model with img2img pipeline. + + Args: + omni: The Omni instance to use for generation. + input_image: The input PIL Image for img2img. + prompt: The text prompt for image editing. + + Returns: + The generated PIL Image. + + Raises: + AssertionError: If no image is generated or size is incorrect. + """ + params_list = _configure_sampling_params(omni) + + omni_outputs = list( + omni.generate( + prompts=[ + { + "prompt": prompt, + "multi_modal_data": {"img2img": input_image}, + "modalities": ["img2img"], + } + ], + sampling_params_list=params_list, + ) + ) + + generated_image = _extract_generated_image(omni_outputs) + assert generated_image is not None, "No images generated" + assert generated_image.size == EXPECTED_OUTPUT_SIZE, f"Expected {EXPECTED_OUTPUT_SIZE}, got {generated_image.size}" + + return generated_image + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +def test_bagel_img2img_shared_memory_connector(): + """Test Bagel img2img with shared memory connector.""" + input_image = _load_input_image() + config_path = str(Path(__file__).parent / "stage_configs" / "bagel_sharedmemory_ci.yaml") + omni = Omni(model="ByteDance-Seed/BAGEL-7B-MoT", stage_configs_path=config_path, stage_init_timeout=300) + + try: + generated_image = _generate_bagel_img2img(omni, input_image) + _validate_pixels(generated_image) + finally: + omni.close() diff --git a/tests/e2e/offline_inference/test_bagel_text2img.py b/tests/e2e/offline_inference/test_bagel_text2img.py index 20f788895..360d49bb1 100644 --- a/tests/e2e/offline_inference/test_bagel_text2img.py +++ b/tests/e2e/offline_inference/test_bagel_text2img.py @@ -14,10 +14,6 @@ Equivalent to running: """ import os - -os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" - import signal import socket import subprocess @@ -37,23 +33,23 @@ from vllm_omni.entrypoints.omni import Omni # "Generated with seed=52, num_inference_steps=15, # prompt='A futuristic city skyline at twilight, cyberpunk style'" REFERENCE_PIXELS = [ - {"position": (100, 100), "rgb": (49, 96, 134)}, - {"position": (400, 50), "rgb": (63, 127, 167)}, - {"position": (700, 100), "rgb": (70, 101, 141)}, - {"position": (150, 400), "rgb": (115, 90, 150)}, - {"position": (512, 512), "rgb": (98, 86, 119)}, - {"position": (700, 400), "rgb": (29, 42, 91)}, - {"position": (100, 700), "rgb": (47, 50, 88)}, - {"position": (400, 700), "rgb": (36, 52, 91)}, - {"position": (700, 700), "rgb": (45, 58, 99)}, - {"position": (256, 256), "rgb": (62, 94, 135)}, + {"position": (100, 100), "rgb": (121, 118, 100)}, + {"position": (400, 50), "rgb": (163, 162, 143)}, + {"position": (700, 100), "rgb": (170, 156, 127)}, + {"position": (150, 400), "rgb": (129, 127, 112)}, + {"position": (512, 512), "rgb": (135, 61, 59)}, + {"position": (700, 400), "rgb": (205, 107, 43)}, + {"position": (100, 700), "rgb": (197, 177, 157)}, + {"position": (400, 700), "rgb": (139, 107, 86)}, + {"position": (700, 700), "rgb": (247, 205, 146)}, + {"position": (256, 256), "rgb": (171, 160, 153)}, ] # Maximum allowed difference per color channel PIXEL_TOLERANCE = 5 # Default test prompt -DEFAULT_PROMPT = "<|im_start|>A futuristic city skyline at twilight, cyberpunk style<|im_end|>" +DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>" def _find_free_port() -> int: diff --git a/tests/e2e/online_serving/test_bagel_online.py b/tests/e2e/online_serving/test_bagel_online.py new file mode 100644 index 000000000..4056cfdef --- /dev/null +++ b/tests/e2e/online_serving/test_bagel_online.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +End-to-end online serving test for Bagel text2img and img2img generation. + +This test validates that the Bagel model can serve image generation requests +via the OpenAI-compatible chat completions API. + +Equivalent to running: + vllm-omni serve "ByteDance-Seed/BAGEL-7B-MoT" --omni --port 8091 + + # text2img + python3 examples/online_serving/bagel/openai_chat_client.py \\ + --prompt "A cute cat" --modality text2img + + # img2img + python3 examples/online_serving/bagel/openai_chat_client.py \\ + --prompt "Let the woman wear a blue dress" --modality img2img \\ + --image-url women.jpg +""" + +import base64 +import os +import signal +import socket +import subprocess +import sys +import time +from io import BytesIO +from pathlib import Path +from typing import Any + +import pytest +import requests +from PIL import Image +from vllm.assets.image import ImageAsset + +from tests.utils import hardware_test + +MODEL = "ByteDance-Seed/BAGEL-7B-MoT" +STAGE_CONFIGS_PATH = str( + Path(__file__).parent.parent / "offline_inference" / "stage_configs" / "bagel_sharedmemory_ci.yaml" +) + +TEXT2IMG_PROMPT = "A cute cat" +IMG2IMG_PROMPT = "Change the grass color to red" + + +class BagelOmniServer: + """Context manager to start/stop a vLLM-Omni server for Bagel model tests.""" + + def __init__( + self, + model: str = MODEL, + stage_configs_path: str = STAGE_CONFIGS_PATH, + env_dict: dict[str, str] | None = None, + ) -> None: + self.model = model + self.stage_configs_path = stage_configs_path + self.env_dict = env_dict + self.proc: subprocess.Popen | None = None + self.host = "127.0.0.1" + self.port = _find_free_port() + + @property + def base_url(self) -> str: + return f"http://{self.host}:{self.port}" + + def _start_server(self) -> None: + env = os.environ.copy() + if self.env_dict is not None: + env.update(self.env_dict) + + cmd = [ + sys.executable, + "-m", + "vllm_omni.entrypoints.cli.main", + "serve", + self.model, + "--omni", + "--host", + self.host, + "--port", + str(self.port), + "--stage-configs-path", + self.stage_configs_path, + "--stage-init-timeout", + "300", + ] + + self.proc = subprocess.Popen( + cmd, + env=env, + start_new_session=True, + ) + + try: + if not _wait_for_port(self.host, self.port, timeout=600, proc=self.proc): + self.terminate() + raise RuntimeError(f"Server failed to start within 600 seconds on {self.host}:{self.port}") + except Exception: + self.terminate() + raise + + def __enter__(self): + self._start_server() + return self + + def terminate(self) -> None: + if self.proc: + try: + os.killpg(os.getpgid(self.proc.pid), signal.SIGTERM) + except ProcessLookupError: + pass + try: + self.proc.wait(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(self.proc.pid), signal.SIGKILL) + except ProcessLookupError: + pass + self.proc.wait() + self.proc = None + + def __exit__(self, exc_type, exc_val, exc_tb): + self.terminate() + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + s.listen(1) + return s.getsockname()[1] + + +def _wait_for_port(host: str, port: int, timeout: int = 600, proc: subprocess.Popen | None = None) -> bool: + start = time.time() + while time.time() - start < timeout: + if proc is not None and proc.poll() is not None: + # Server process exited early + return False + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + if sock.connect_ex((host, port)) == 0: + return True + except Exception: + pass + time.sleep(2) + return False + + +def _send_chat_request( + server_url: str, + prompt: str, + *, + modality: str = "text2img", + image: Image.Image | None = None, + timeout: int = 300, +) -> dict[str, Any]: + """Send a chat completion request matching the openai_chat_client.py format.""" + content: list[dict[str, Any]] = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}] + + if image is not None: + buffer = BytesIO() + image.save(buffer, format="JPEG") + b64_data = base64.b64encode(buffer.getvalue()).decode("utf-8") + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{b64_data}"}, + } + ) + + payload: dict[str, Any] = { + "messages": [{"role": "user", "content": content}], + } + + if modality in ("text2img", "img2img"): + payload["modalities"] = ["image"] + + resp = requests.post( + f"{server_url}/v1/chat/completions", + headers={"Content-Type": "application/json"}, + json=payload, + timeout=timeout, + ) + resp.raise_for_status() + return resp.json() + + +def _extract_image_from_response(data: dict[str, Any]) -> Image.Image | None: + """Extract the generated PIL Image from a chat completion response.""" + for choice in data.get("choices", []): + content = choice.get("message", {}).get("content") + if isinstance(content, list) and content: + first_item = content[0] + if isinstance(first_item, dict) and "image_url" in first_item: + url = first_item["image_url"].get("url", "") + if url.startswith("data:image"): + _, b64 = url.split(",", 1) + return Image.open(BytesIO(base64.b64decode(b64))) + return None + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +def test_bagel_text2img_online(): + """Test Bagel text2img via OpenAI-compatible chat completions API.""" + with BagelOmniServer() as server: + response_data = _send_chat_request( + server.base_url, + TEXT2IMG_PROMPT, + modality="text2img", + ) + + image = _extract_image_from_response(response_data) + assert image is not None, f"No image in response: {response_data}" + image.load() + + w, h = image.size + assert w > 0 and h > 0, f"Invalid image size: {image.size}" + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +def test_bagel_img2img_online(): + """Test Bagel img2img via OpenAI-compatible chat completions API.""" + input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB") + + with BagelOmniServer() as server: + response_data = _send_chat_request( + server.base_url, + IMG2IMG_PROMPT, + modality="img2img", + image=input_image, + ) + + image = _extract_image_from_response(response_data) + assert image is not None, f"No image in response: {response_data}" + image.load() + + w, h = image.size + assert w > 0 and h > 0, f"Invalid image size: {image.size}"