[Bugfix] Honor the requested output canvas in BAGEL img2img (#7287)

Signed-off-by: Ding Zuhao <e1583181@u.nus.edu>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Ding ZuhaoandCursor authored and GitHub committed 2026-09-21 07:29:24 +00:00
1 parent 86fcfe95f1
commit 154a1f25f5
8 files changed
+391 -27

No files matched your search

@@ -35,6 +35,7 @@ from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.errors import OmniClientError
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from .autoencoder import AutoEncoder, AutoEncoderParams, DistributedAutoEncoder
@@ -184,6 +185,45 @@ def _bagel_effective_image_size(
return min(resized_height, max_image_size), min(resized_width, max_image_size)
def _bagel_canvas_requested(sampling: OmniDiffusionSamplingParams) -> bool:
return (sampling.height is not None and not sampling.height_not_provided) or (
sampling.width is not None and not sampling.width_not_provided
)
def _bagel_align_requested_canvas(
sampling: OmniDiffusionSamplingParams,
*,
latent_downsample: int,
max_latent_size: int,
) -> None:
"""Floor an explicitly requested canvas to the latent stride; reject sides over the checkpoint limit."""
max_image_size = int(max_latent_size * latent_downsample)
resized: list[str] = []
for name in ("width", "height"):
value = getattr(sampling, name)
if value is None or getattr(sampling, f"{name}_not_provided"):
continue
value = int(value)
if value > max_image_size:
raise OmniClientError(
f"Requested {name}={value} exceeds the BAGEL checkpoint limit of {max_image_size} "
f"(max_latent_size={max_latent_size}, latent_downsample={latent_downsample})."
)
aligned = max(latent_downsample, (value // latent_downsample) * latent_downsample)
if aligned != value:
setattr(sampling, name, aligned)
resized.append(f"{name} {value}->{aligned}")
if resized:
logger.warning(
"BAGEL: requested canvas is not a multiple of the latent stride %d and cannot be "
"generated as-is; resizing %s.",
latent_downsample,
", ".join(resized),
)
def get_bagel_pre_process_func(od_config: OmniDiffusionConfig):
"""Resolve BAGEL execution mode and step-batch compatibility."""
@@ -194,12 +234,23 @@ def get_bagel_pre_process_func(od_config: OmniDiffusionConfig):
nonlocal image_geometry
sampling = request.sampling_params
modalities = (request.prompt.get("modalities") or []) if isinstance(request.prompt, dict) else []
kv_metadata = getattr(sampling, "kv_metadata", None) or {}
image_shape = kv_metadata.get("image_shape")
if image_shape is not None:
if _bagel_canvas_requested(sampling) and "text" not in modalities:
if image_geometry is None:
image_geometry = _resolve_bagel_image_geometry(od_config)
latent_downsample, max_latent_size = image_geometry
_bagel_align_requested_canvas(
sampling,
latent_downsample=latent_downsample,
max_latent_size=max_latent_size,
)
elif image_shape is not None:
sampling.height, sampling.width = (int(value) for value in image_shape)
sampling.height_not_provided = False
sampling.width_not_provided = False
elif isinstance(request.prompt, dict):
modalities = request.prompt.get("modalities") or []
multi_modal_data = request.prompt.get("multi_modal_data") or {}
image_input = multi_modal_data.get("img2img")
if image_input is None and "text" not in modalities:
@@ -220,6 +271,8 @@ def get_bagel_pre_process_func(od_config: OmniDiffusionConfig):
latent_downsample=latent_downsample,
max_latent_size=max_latent_size,
)
sampling.height_not_provided = True
sampling.width_not_provided = True
extra_args = request.sampling_params.extra_args or {}
cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0)))
@@ -528,6 +581,7 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf
f"latent_downsample={self.bagel.latent_downsample})."
)
image_shape = (height, width)
image_shape_requested = _bagel_canvas_requested(sampling)
extra_args = getattr(sampling, "extra_args", {}) or {}
cfg_text_scale = extra_args.get("cfg_text_scale", 4.0)
@@ -568,7 +622,15 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf
gen_context["ropes"] = [seq_len]
if sampling.kv_metadata and "image_shape" in sampling.kv_metadata:
image_shape = tuple(sampling.kv_metadata["image_shape"])
if not image_shape_requested:
image_shape = tuple(sampling.kv_metadata["image_shape"])
elif tuple(sampling.kv_metadata["image_shape"]) != image_shape:
logger.info(
"Generating at requested %dx%d instead of KV image shape %s",
image_shape[1],
image_shape[0],
sampling.kv_metadata["image_shape"],
)
branch_kvs = getattr(sampling, "cfg_branch_past_key_values", None) or {}
branch_metadata = getattr(sampling, "cfg_branch_kv_metadata", None) or {}
@@ -673,8 +735,15 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf
image_input = [_resize_to_stride(img) for img in image_input]
resized_w, resized_h = image_input[0].size
image_shape = (resized_h, resized_w)
logger.info(f"img2img: resized image to {resized_w}x{resized_h}")
if not image_shape_requested:
image_shape = (resized_h, resized_w)
logger.info(
"img2img: resized source to %dx%d, generating at %dx%d",
resized_w,
resized_h,
image_shape[1],
image_shape[0],
)
def vae_transforms(img):
if img.mode != "RGB":