[Bugfix] Fix Bagel online mode for 1. Hang after several requests 2. Non-deterministic image quality regression. (#2458)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2026-04-13 03:36:20 +00:00
1 parent eb1a801b21
commit e12250119b
4 files changed
+164 -197

No files matched your search

+54 -51
View File
@@ -59,6 +59,11 @@ class OmniARScheduler(VLLMScheduler):
# Track ACTIVE transfers (submitted to runner but not yet acked via kv_extracted_req_ids)
self.active_kv_transfers: set[str] = set()
# Requests marked for deferred stop: keep running until KV extraction
# completes so that kv_ready can be emitted while the request is still
# alive. Stopped on the first scheduler step after extraction ack.
self.pending_stop_after_extraction: set[str] = set()
# [Omni] Pre-parse KV transfer criteria
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
@@ -126,11 +131,16 @@ class OmniARScheduler(VLLMScheduler):
stop_decode_on_trigger = self.kv_transfer_criteria.get("stop_after_transfer", True)
if request.request_id in self.transfer_triggered_requests:
# Already triggered. When stop_decode_on_trigger is True AND
# transfer was actually queued, the request was already stopped
# at trigger time (see below). Any request that reaches this
# point either has stop_decode_on_trigger=False (continue
# decoding) or was not actually queued (should not be stopped).
# Deferred stop: once KV extraction is complete (no longer in
# active_kv_transfers), stop the request. This guarantees the
# kv_ready signal was emitted while the request was still alive.
if (
request.request_id in self.pending_stop_after_extraction
and request.request_id not in self.active_kv_transfers
):
self.pending_stop_after_extraction.discard(request.request_id)
request.status = RequestStatus.FINISHED_STOPPED
return True
return False
if criteria_type == "prefill_finished":
@@ -140,14 +150,11 @@ class OmniARScheduler(VLLMScheduler):
actually_queued = request.request_id in self.requests_needing_kv_transfer
if stop_decode_on_trigger and actually_queued:
# Stop immediately so the request is NOT scheduled in
# the next step, freeing scheduling budget for companion
# requests whose chunked-prefill boundaries must be
# deterministic. waiting_for_transfer_free keeps blocks
# alive until the model runner finishes KV extraction.
self.waiting_for_transfer_free.add(request.request_id)
request.status = RequestStatus.FINISHED_STOPPED
return True
# Defer the stop until KV extraction completes so that
# the kv_ready signal can be emitted while the request
# is still alive. The request will be stopped on the
# next scheduler step after extraction ack arrives.
self.pending_stop_after_extraction.add(request.request_id)
return False
@@ -167,9 +174,7 @@ class OmniARScheduler(VLLMScheduler):
actually_queued = request.request_id in self.requests_needing_kv_transfer
if stop_decode_on_trigger and actually_queued:
self.waiting_for_transfer_free.add(request.request_id)
request.status = RequestStatus.FINISHED_STOPPED
return True
self.pending_stop_after_extraction.add(request.request_id)
return False
@@ -268,6 +273,26 @@ class OmniARScheduler(VLLMScheduler):
num_scheduled_tokens,
)
# Pre-process KV extraction acks so that the per-request loop below
# can see up-to-date active_kv_transfers state and emit kv_ready
# signals while requests are still alive (before any deferred stop).
kv_extracted_ids = getattr(model_runner_output, "kv_extracted_req_ids", None)
if kv_extracted_ids:
for req_id in kv_extracted_ids:
try:
self.active_kv_transfers.discard(req_id)
req = self.requests.get(req_id)
if req is not None and not req.is_finished():
outputs[req.client_index].append(
EngineCoreOutput(
request_id=req_id,
new_token_ids=[],
kv_transfer_params={"kv_ready": True},
)
)
except Exception:
init_logger(__name__).exception("Failed to pre-process KV extraction for %s", req_id)
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
# the below loop can be a performance bottleneck. We should do our best
# to avoid expensive operations inside the loop.
@@ -436,6 +461,7 @@ class OmniARScheduler(VLLMScheduler):
self.transfer_triggered_requests.remove(req.request_id)
if req.request_id in self.active_kv_transfers:
self.active_kv_transfers.remove(req.request_id)
self.pending_stop_after_extraction.discard(req.request_id)
# Same for preempted
for req in stopped_preempted_reqs:
@@ -444,6 +470,8 @@ class OmniARScheduler(VLLMScheduler):
self.transfer_triggered_requests.remove(req.request_id)
if req.request_id in self.active_kv_transfers:
self.active_kv_transfers.remove(req.request_id)
self.pending_stop_after_extraction.discard(req.request_id)
# KV Connector: update state for finished KV Transfers.
if kv_connector_output:
self._update_from_kv_xfer_finished(kv_connector_output)
@@ -489,35 +517,12 @@ class OmniARScheduler(VLLMScheduler):
engine_core_outputs[0] = eco = EngineCoreOutputs()
eco.scheduler_stats = stats
# This is where we free blocks that were held for transfer
try:
kv_extracted_ids = getattr(model_runner_output, "kv_extracted_req_ids", None)
if kv_extracted_ids:
for req_id in kv_extracted_ids:
# Emit a kv_ready signal so the orchestrator can forward
# the request to the DiT stage immediately after KV
# extraction, without waiting for AR decode to finish.
req = self.requests.get(req_id)
if req is not None and not req.is_finished():
eco = engine_core_outputs.get(req.client_index)
if eco is None:
eco = EngineCoreOutputs()
engine_core_outputs[req.client_index] = eco
eco.outputs.append(
EngineCoreOutput(
request_id=req_id,
new_token_ids=[],
kv_transfer_params={"kv_ready": True},
)
)
# Mark transfer as finished
if req_id in self.active_kv_transfers:
self.active_kv_transfers.remove(req_id)
logger.debug(f"[Omni] KV Transfer finished for {req_id}")
# Free blocks that were held for transfer (kv_ready and
# active_kv_transfers updates already done before the per-request loop).
if kv_extracted_ids:
for req_id in kv_extracted_ids:
try:
if req_id in self.waiting_for_transfer_free:
# Now it's safe to free blocks
req = self.requests.get(req_id)
if req:
self.kv_cache_manager.free(req)
@@ -525,13 +530,12 @@ class OmniARScheduler(VLLMScheduler):
del self.requests[req_id]
if req_id in self.transfer_triggered_requests:
self.transfer_triggered_requests.remove(req_id)
if req_id in self.active_kv_transfers:
self.active_kv_transfers.remove(req_id)
self.active_kv_transfers.discard(req_id)
self.pending_stop_after_extraction.discard(req_id)
logger.debug(f"Freed blocks for {req_id} after transfer extraction")
self.waiting_for_transfer_free.remove(req_id)
except Exception:
init_logger(__name__).exception("Failed to process finished transfer requests")
except Exception:
init_logger(__name__).exception("Failed to free blocks for %s after transfer", req_id)
return engine_core_outputs
@@ -564,8 +568,7 @@ class OmniARScheduler(VLLMScheduler):
kv_xfer_params = None
return kv_xfer_params
elif request_id in self.waiting_for_transfer_free:
# Stopped immediately by stop_decode_on_trigger; blocks are
# held until KV extraction completes in a future step.
# Blocks held until KV extraction completes in a future step.
return None
else:
logger.debug(
+63 -132
View File
@@ -1,4 +1,3 @@
from collections import deque
from collections.abc import Iterable, Mapping, Sequence
from math import isqrt
from typing import Any
@@ -442,14 +441,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
self._pending_img2img_info: list[tuple[int, int, int, int]] = []
self._ropes_pending: list[dict[str, Any]] = []
self._ropes_metadata: dict[str, dict[str, Any]] = {}
self._cfg_companion_queue: deque[tuple[tuple[int, int, int, int], int]] = deque()
# Per-request position offset for decode after img2img prefill.
# Prefill rewrites positions (VAE→0, ViT→1, text→2..N) but the model
# runner assigns decode positions starting from prefill_len, not N+1.
# offset = rope - prefill_len (a negative number).
self._pending_decode_offsets: list[int] = []
self._decode_position_offsets: dict[str, int] = {}
self._last_img2img_info: tuple[int, int, int, int] | None = None
from transformers import AutoTokenizer
@@ -461,7 +453,6 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
self._start_of_image_id = int(_tok.convert_tokens_to_ids("<|vision_start|>"))
self._end_of_image_id = int(_tok.convert_tokens_to_ids("<|vision_end|>"))
self._img2img_token_id = int(_tok.convert_tokens_to_ids("<|fim_middle|>"))
self._vae_token_mask: torch.Tensor | None = None
self.device = get_local_device()
self._install_mot_modules(config)
@@ -540,9 +531,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
self._ropes_pending.clear()
self._ropes_metadata.clear()
self._pending_img2img_info.clear()
self._cfg_companion_queue.clear()
self._pending_decode_offsets.clear()
self._decode_position_offsets.clear()
self._last_img2img_info = None
self._vae_token_mask = None
def get_kv_transfer_metadata(
@@ -554,12 +543,10 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
meta = self._ropes_metadata.pop(req_id, None)
if meta is None:
return None
# In think-mode img2img the prefill rope doesn't account for decoded
# thinking tokens; correct it to num_computed_tokens + offset.
# Skip correction when num_computed_tokens is unavailable (None).
offset = self._decode_position_offsets.pop(req_id, 0)
if offset != 0 and "ropes" in meta and num_computed_tokens is not None:
meta["ropes"] = [num_computed_tokens + offset]
if num_computed_tokens is not None and "image_shape" in meta:
prefill_rope = meta["ropes"][0] if meta.get("ropes") else 0
if num_computed_tokens > prefill_rope:
meta["ropes"] = [num_computed_tokens]
return meta
def prepare_runner_inputs(
@@ -572,48 +559,29 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
num_scheduled_tokens: list[int],
input_ids_buffer: torch.Tensor | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
"""Model-runner hook: adjust inputs before ``forward()``.
Returns ``(input_ids, positions)`` — possibly modified.
Two adjustments for BAGEL img2img:
1. **Restore input_ids** when ``inputs_embeds`` is present so that
``_adjust_positions_for_img2img`` can locate the
``<|fim_middle|>`` placeholder.
2. **Decode position offset**: prefill rewrites positions to a
compact scheme (rope ≪ prefill_len). The runner assigns decode
positions from ``num_computed_tokens``, which is far too large;
apply the stored per-request offset.
"""
"""Restore input_ids so _adjust_positions_for_img2img can locate
the <|fim_middle|> placeholder for thinking-mode pre_text_len
detection."""
if inputs_embeds is not None and input_ids is None and input_ids_buffer is not None:
input_ids = input_ids_buffer
if self._decode_position_offsets and positions is not None:
token_start = 0
for i, rid in enumerate(req_ids):
sched = num_scheduled_tokens[i]
offset = self._decode_position_offsets.get(rid, 0)
if offset != 0 and num_computed_tokens[i] > 0:
positions[token_start : token_start + sched] += offset
token_start += sched
return input_ids, positions
def flush_pending_metadata(self, req_ids: list[str]) -> None:
"""Map pending metadata (batch order) to req_ids after forward()."""
"""Map pending metadata (batch order) to req_ids after forward().
Guard: if a request already has metadata with ``image_shape``
(written during img2img prefill), don't overwrite it with
decode-step metadata that lacks ``image_shape``.
"""
pending = self._ropes_pending
self._ropes_pending = []
for i, meta in enumerate(pending):
if i < len(req_ids):
if req_ids[i] not in self._ropes_metadata:
self._ropes_metadata[req_ids[i]] = meta
pending_offsets = self._pending_decode_offsets
self._pending_decode_offsets = []
for i, offset in enumerate(pending_offsets):
if i < len(req_ids) and offset != 0:
self._decode_position_offsets[req_ids[i]] = offset
rid = req_ids[i]
existing = self._ropes_metadata.get(rid)
if existing and "image_shape" in existing and "image_shape" not in meta:
continue
self._ropes_metadata[rid] = meta
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
@@ -727,16 +695,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
num_vit = vit_emb.shape[0] + 2
info = (num_vae, num_vit, int(H), int(W))
self._pending_img2img_info.append(info)
# Only the gen (main) request should add a companion queue entry.
# Companion requests (cfg_text, cfg_img) also call this method with
# the same image, so guard by checking whether this exact info
# tuple is already enqueued. For batched img2img with multiple
# concurrent gen requests this correctly adds one entry per unique
# image; images with identical (num_vae, num_vit, H, W) that arrive
# in the same batch are indistinguishable here and will share one
# entry, but that is an uncommon edge case.
if not any(entry[0] == info for entry in self._cfg_companion_queue):
self._cfg_companion_queue.append((info, 2)) # cfg_text + cfg_img
self._last_img2img_info = info
return tuple(results)
@@ -755,31 +714,18 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
positions = self._adjust_positions_for_img2img(positions, input_ids)
use_mot = True
elif self._cfg_companion_queue:
# Guard: if this looks like a pure decode step (small token count,
# no multimodal embeddings), the queue has stale entries from a
# previous prefill cycle — clear them instead of consuming.
if inputs_embeds is None and seq_len <= 2:
self._cfg_companion_queue.clear()
elif self._last_img2img_info is not None:
info = self._last_img2img_info
num_vae, num_vit, _, _ = info
num_img2img = num_vae + 1 + num_vit
if seq_len >= num_img2img:
self._pending_img2img_info = [info]
positions = self._adjust_positions_for_img2img(positions, input_ids)
use_mot = True
else:
cached, remaining = self._cfg_companion_queue[0]
remaining -= 1
num_vae, num_vit, img_H, img_W = cached
num_img2img = num_vae + 1 + num_vit # +1 separator
seq_len = inputs_embeds.shape[0] if inputs_embeds is not None else positions.shape[0]
if inputs_embeds is not None and seq_len >= num_img2img:
self._pending_img2img_info = [cached]
positions = self._adjust_positions_for_img2img(positions, input_ids)
use_mot = True
else:
rope = int(positions[seq_len - 1].item()) + 1
self._ropes_pending.append({"ropes": [rope]})
if remaining == 0:
self._cfg_companion_queue.popleft()
else:
self._cfg_companion_queue[0] = (cached, remaining)
rope = int(positions[seq_len - 1].item()) + 1
self._ropes_pending.append({"ropes": [rope]})
if use_mot:
return self._mot_forward(input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs)
@@ -790,27 +736,18 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
positions: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
"""Rewrite position IDs to match the original BAGEL position scheme:
"""Rewrite position IDs for img2img.
If there are ``pre_text_len`` text tokens before the img2img block::
Supports an optional ``pre_text_len`` prefix (thinking-mode) detected
via the ``<|fim_middle|>`` token in *input_ids*:
pre_text → 0, 1, ..., M-1
VAE → M (all share)
separator→ M
ViT → M+1 (all share)
post_text→ M+2, M+3, ...
pre_text -> 0 .. M-1
VAE -> M (all share)
separator-> M
ViT -> M+1 (all share)
post_text-> M+2, M+3, ...
When no text precedes the img2img block (M=0), this reduces to the
simpler scheme: VAE→0, ViT→1, text→2, 3, ...
Also computes ``self._vae_token_mask`` (bool tensor, True for actual
VAE latent patches that should use gen-mode weights) and pushes
per-request ropes + image_shape to the FIFO consumed by
``get_kv_transfer_metadata``.
For img2img requests, also stores a decode position offset so that
subsequent autoregressive decode steps use positions that continue
from the rewritten scheme rather than from the original prefill length.
When M=0 (standard img2img) this reduces to VAE->0, ViT->1, text->2..
"""
info_list = self._pending_img2img_info
self._pending_img2img_info = []
@@ -836,70 +773,64 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
req_len = end - start
if img2img_idx < len(info_list):
num_vae, num_vit, img_H, img_W = info_list[img2img_idx]
cur_info = info_list[img2img_idx]
elif self._last_img2img_info is not None:
cur_info = self._last_img2img_info
else:
cur_info = None
if cur_info is not None:
num_vae, num_vit, img_H, img_W = cur_info
num_img2img = num_vae + 1 + num_vit # +1 separator
if req_len >= num_img2img:
# Detect offset of img2img tokens within this request
# by searching for the img2img placeholder token ID.
pre_text_len = 0
if input_ids is not None:
req_ids = input_ids[start:end]
mask = req_ids == self._img2img_token_id
indices = mask.nonzero(as_tuple=True)[0]
req_ids_slice = input_ids[start:end]
indices = (req_ids_slice == self._img2img_token_id).nonzero(as_tuple=True)[0]
if indices.numel() > 0:
pre_text_len = int(indices[0].item())
img_start = start + pre_text_len
M = pre_text_len
img_start = start + M
post_text_start = img_start + num_img2img
# pre_text_pos: position base for image tokens
pre_text_pos = pre_text_len
# Pre-image text: sequential positions 0..pre_text_pos-1
if pre_text_len > 0:
if M > 0:
new_positions[start:img_start] = torch.arange(
0, pre_text_pos, device=positions.device, dtype=positions.dtype
0, M, device=positions.device, dtype=positions.dtype
)
# VAE tokens: all share position pre_text_pos
new_positions[img_start : img_start + num_vae] = pre_text_pos
# Separator: position pre_text_pos
new_positions[img_start + num_vae] = pre_text_pos
# ViT tokens: all share position pre_text_pos+1
new_positions[img_start : img_start + num_vae] = M
new_positions[img_start + num_vae] = M # separator
vit_start = img_start + num_vae + 1
new_positions[vit_start : vit_start + num_vit] = pre_text_pos + 1
new_positions[vit_start : vit_start + num_vit] = M + 1
# Post-image text: sequential positions pre_text_pos+2, pre_text_pos+3, ...
num_post_text = end - post_text_start
if num_post_text > 0:
new_positions[post_text_start:end] = torch.arange(
pre_text_pos + 2,
pre_text_pos + 2 + num_post_text,
M + 2,
M + 2 + num_post_text,
device=positions.device,
dtype=positions.dtype,
)
# VAE gen-mode mask: only actual VAE latent patches (not markers)
vae_patches_start = img_start + 1 # skip start_marker
vae_patches_end = img_start + num_vae - 1 # before end_marker
vae_patches_start = img_start + 1
vae_patches_end = img_start + num_vae - 1
if vae_patches_end > vae_patches_start:
vae_mask[vae_patches_start:vae_patches_end] = True
rope = pre_text_pos + 2 + num_post_text
rope = M + 2 + num_post_text
self._ropes_pending.append(
{
"ropes": [rope],
"image_shape": [img_H, img_W],
}
)
decode_offset = rope - req_len
self._pending_decode_offsets.append(decode_offset)
img2img_idx += 1
continue
rope = int(new_positions[end - 1].item()) + 1
self._ropes_pending.append({"ropes": [rope]})
self._pending_decode_offsets.append(0)
self._vae_token_mask = vae_mask if vae_mask.any() else None
return new_positions
@@ -149,7 +149,15 @@ class NPUARModelRunner(OmniNPUModelRunner):
encoder_cache=self.encoder_cache,
) as ec_connector_output:
self._execute_mm_encoder(scheduler_output)
return make_empty_encoder_model_runner_output(scheduler_output)
kv_ids = self.kv_extracted_req_ids
self.kv_extracted_req_ids = None
output = make_empty_encoder_model_runner_output(scheduler_output)
if kv_ids:
output = copy(output)
output.kv_extracted_req_ids = kv_ids
return output
if not num_scheduled_tokens:
if (
@@ -163,10 +171,20 @@ class NPUARModelRunner(OmniNPUModelRunner):
# dummy run to ensure coordinate_batch_across_dp
# is called into to avoid out of sync issues.
self._dummy_run(1)
kv_ids = self.kv_extracted_req_ids
self.kv_extracted_req_ids = None
if not has_kv_transfer_group():
# Return empty ModelRunnerOutput if no work to do.
return EMPTY_MODEL_RUNNER_OUTPUT
return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
output = EMPTY_MODEL_RUNNER_OUTPUT
else:
output = self.kv_connector_no_forward(scheduler_output, self.vllm_config)
if kv_ids:
output = copy(output)
output.kv_extracted_req_ids = kv_ids
return output
if self.cache_config.kv_sharing_fast_prefill:
assert not self.num_prompt_logprobs, (
"--kv-sharing-fast-prefill produces incorrect "
+25 -10
View File
@@ -205,24 +205,39 @@ class GPUARModelRunner(OmniGPUModelRunner):
encoder_cache=self.encoder_cache,
) as ec_connector_output:
self._execute_mm_encoder(scheduler_output)
return make_empty_encoder_model_runner_output(scheduler_output)
kv_ids = self.kv_extracted_req_ids
self.kv_extracted_req_ids = None
output = make_empty_encoder_model_runner_output(scheduler_output)
if kv_ids:
output = copy(output)
output.kv_extracted_req_ids = kv_ids
return output
if not num_scheduled_tokens:
if (
self.parallel_config.distributed_executor_backend == "external_launcher"
and self.parallel_config.data_parallel_size > 1
):
# this is a corner case when both external launcher
# and DP are enabled, num_scheduled_tokens could be
# 0, and has_unfinished_requests in the outer loop
# returns True. before returning early here we call
# dummy run to ensure coordinate_batch_across_dp
# is called into to avoid out of sync issues.
self._dummy_run(1)
# Capture KV extraction results before early return;
# sample_tokens() is skipped on this path so the IDs
# would otherwise be silently overwritten next step.
kv_ids = self.kv_extracted_req_ids
self.kv_extracted_req_ids = None
if not has_kv_transfer_group():
# Return empty ModelRunnerOutput if no work to do.
return EMPTY_MODEL_RUNNER_OUTPUT
return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
output = EMPTY_MODEL_RUNNER_OUTPUT
else:
output = self.kv_connector_no_forward(scheduler_output, self.vllm_config)
if kv_ids:
output = copy(output)
output.kv_extracted_req_ids = kv_ids
return output
if self.cache_config.kv_sharing_fast_prefill:
assert not self.num_prompt_logprobs, (