[Bugfix] Fix Bagel online mode for 1. Hang after several requests 2. Non-deterministic image quality regression. (#2458)
Signed-off-by: natureofnature <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
eb1a801b21
commit
e12250119b
4 files changed
+164
-197
No files matched your search
@@ -59,6 +59,11 @@ class OmniARScheduler(VLLMScheduler):
|
||||
# Track ACTIVE transfers (submitted to runner but not yet acked via kv_extracted_req_ids)
|
||||
self.active_kv_transfers: set[str] = set()
|
||||
|
||||
# Requests marked for deferred stop: keep running until KV extraction
|
||||
# completes so that kv_ready can be emitted while the request is still
|
||||
# alive. Stopped on the first scheduler step after extraction ack.
|
||||
self.pending_stop_after_extraction: set[str] = set()
|
||||
|
||||
# [Omni] Pre-parse KV transfer criteria
|
||||
self.kv_transfer_criteria = self._get_kv_transfer_criteria()
|
||||
|
||||
@@ -126,11 +131,16 @@ class OmniARScheduler(VLLMScheduler):
|
||||
stop_decode_on_trigger = self.kv_transfer_criteria.get("stop_after_transfer", True)
|
||||
|
||||
if request.request_id in self.transfer_triggered_requests:
|
||||
# Already triggered. When stop_decode_on_trigger is True AND
|
||||
# transfer was actually queued, the request was already stopped
|
||||
# at trigger time (see below). Any request that reaches this
|
||||
# point either has stop_decode_on_trigger=False (continue
|
||||
# decoding) or was not actually queued (should not be stopped).
|
||||
# Deferred stop: once KV extraction is complete (no longer in
|
||||
# active_kv_transfers), stop the request. This guarantees the
|
||||
# kv_ready signal was emitted while the request was still alive.
|
||||
if (
|
||||
request.request_id in self.pending_stop_after_extraction
|
||||
and request.request_id not in self.active_kv_transfers
|
||||
):
|
||||
self.pending_stop_after_extraction.discard(request.request_id)
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
return True
|
||||
return False
|
||||
|
||||
if criteria_type == "prefill_finished":
|
||||
@@ -140,14 +150,11 @@ class OmniARScheduler(VLLMScheduler):
|
||||
actually_queued = request.request_id in self.requests_needing_kv_transfer
|
||||
|
||||
if stop_decode_on_trigger and actually_queued:
|
||||
# Stop immediately so the request is NOT scheduled in
|
||||
# the next step, freeing scheduling budget for companion
|
||||
# requests whose chunked-prefill boundaries must be
|
||||
# deterministic. waiting_for_transfer_free keeps blocks
|
||||
# alive until the model runner finishes KV extraction.
|
||||
self.waiting_for_transfer_free.add(request.request_id)
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
return True
|
||||
# Defer the stop until KV extraction completes so that
|
||||
# the kv_ready signal can be emitted while the request
|
||||
# is still alive. The request will be stopped on the
|
||||
# next scheduler step after extraction ack arrives.
|
||||
self.pending_stop_after_extraction.add(request.request_id)
|
||||
|
||||
return False
|
||||
|
||||
@@ -167,9 +174,7 @@ class OmniARScheduler(VLLMScheduler):
|
||||
actually_queued = request.request_id in self.requests_needing_kv_transfer
|
||||
|
||||
if stop_decode_on_trigger and actually_queued:
|
||||
self.waiting_for_transfer_free.add(request.request_id)
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
return True
|
||||
self.pending_stop_after_extraction.add(request.request_id)
|
||||
|
||||
return False
|
||||
|
||||
@@ -268,6 +273,26 @@ class OmniARScheduler(VLLMScheduler):
|
||||
num_scheduled_tokens,
|
||||
)
|
||||
|
||||
# Pre-process KV extraction acks so that the per-request loop below
|
||||
# can see up-to-date active_kv_transfers state and emit kv_ready
|
||||
# signals while requests are still alive (before any deferred stop).
|
||||
kv_extracted_ids = getattr(model_runner_output, "kv_extracted_req_ids", None)
|
||||
if kv_extracted_ids:
|
||||
for req_id in kv_extracted_ids:
|
||||
try:
|
||||
self.active_kv_transfers.discard(req_id)
|
||||
req = self.requests.get(req_id)
|
||||
if req is not None and not req.is_finished():
|
||||
outputs[req.client_index].append(
|
||||
EngineCoreOutput(
|
||||
request_id=req_id,
|
||||
new_token_ids=[],
|
||||
kv_transfer_params={"kv_ready": True},
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
init_logger(__name__).exception("Failed to pre-process KV extraction for %s", req_id)
|
||||
|
||||
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
|
||||
# the below loop can be a performance bottleneck. We should do our best
|
||||
# to avoid expensive operations inside the loop.
|
||||
@@ -436,6 +461,7 @@ class OmniARScheduler(VLLMScheduler):
|
||||
self.transfer_triggered_requests.remove(req.request_id)
|
||||
if req.request_id in self.active_kv_transfers:
|
||||
self.active_kv_transfers.remove(req.request_id)
|
||||
self.pending_stop_after_extraction.discard(req.request_id)
|
||||
|
||||
# Same for preempted
|
||||
for req in stopped_preempted_reqs:
|
||||
@@ -444,6 +470,8 @@ class OmniARScheduler(VLLMScheduler):
|
||||
self.transfer_triggered_requests.remove(req.request_id)
|
||||
if req.request_id in self.active_kv_transfers:
|
||||
self.active_kv_transfers.remove(req.request_id)
|
||||
self.pending_stop_after_extraction.discard(req.request_id)
|
||||
|
||||
# KV Connector: update state for finished KV Transfers.
|
||||
if kv_connector_output:
|
||||
self._update_from_kv_xfer_finished(kv_connector_output)
|
||||
@@ -489,35 +517,12 @@ class OmniARScheduler(VLLMScheduler):
|
||||
engine_core_outputs[0] = eco = EngineCoreOutputs()
|
||||
eco.scheduler_stats = stats
|
||||
|
||||
# This is where we free blocks that were held for transfer
|
||||
try:
|
||||
kv_extracted_ids = getattr(model_runner_output, "kv_extracted_req_ids", None)
|
||||
if kv_extracted_ids:
|
||||
for req_id in kv_extracted_ids:
|
||||
# Emit a kv_ready signal so the orchestrator can forward
|
||||
# the request to the DiT stage immediately after KV
|
||||
# extraction, without waiting for AR decode to finish.
|
||||
req = self.requests.get(req_id)
|
||||
if req is not None and not req.is_finished():
|
||||
eco = engine_core_outputs.get(req.client_index)
|
||||
if eco is None:
|
||||
eco = EngineCoreOutputs()
|
||||
engine_core_outputs[req.client_index] = eco
|
||||
eco.outputs.append(
|
||||
EngineCoreOutput(
|
||||
request_id=req_id,
|
||||
new_token_ids=[],
|
||||
kv_transfer_params={"kv_ready": True},
|
||||
)
|
||||
)
|
||||
|
||||
# Mark transfer as finished
|
||||
if req_id in self.active_kv_transfers:
|
||||
self.active_kv_transfers.remove(req_id)
|
||||
logger.debug(f"[Omni] KV Transfer finished for {req_id}")
|
||||
|
||||
# Free blocks that were held for transfer (kv_ready and
|
||||
# active_kv_transfers updates already done before the per-request loop).
|
||||
if kv_extracted_ids:
|
||||
for req_id in kv_extracted_ids:
|
||||
try:
|
||||
if req_id in self.waiting_for_transfer_free:
|
||||
# Now it's safe to free blocks
|
||||
req = self.requests.get(req_id)
|
||||
if req:
|
||||
self.kv_cache_manager.free(req)
|
||||
@@ -525,13 +530,12 @@ class OmniARScheduler(VLLMScheduler):
|
||||
del self.requests[req_id]
|
||||
if req_id in self.transfer_triggered_requests:
|
||||
self.transfer_triggered_requests.remove(req_id)
|
||||
if req_id in self.active_kv_transfers:
|
||||
self.active_kv_transfers.remove(req_id)
|
||||
|
||||
self.active_kv_transfers.discard(req_id)
|
||||
self.pending_stop_after_extraction.discard(req_id)
|
||||
logger.debug(f"Freed blocks for {req_id} after transfer extraction")
|
||||
self.waiting_for_transfer_free.remove(req_id)
|
||||
except Exception:
|
||||
init_logger(__name__).exception("Failed to process finished transfer requests")
|
||||
except Exception:
|
||||
init_logger(__name__).exception("Failed to free blocks for %s after transfer", req_id)
|
||||
|
||||
return engine_core_outputs
|
||||
|
||||
@@ -564,8 +568,7 @@ class OmniARScheduler(VLLMScheduler):
|
||||
kv_xfer_params = None
|
||||
return kv_xfer_params
|
||||
elif request_id in self.waiting_for_transfer_free:
|
||||
# Stopped immediately by stop_decode_on_trigger; blocks are
|
||||
# held until KV extraction completes in a future step.
|
||||
# Blocks held until KV extraction completes in a future step.
|
||||
return None
|
||||
else:
|
||||
logger.debug(
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
from collections import deque
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from math import isqrt
|
||||
from typing import Any
|
||||
@@ -442,14 +441,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
self._pending_img2img_info: list[tuple[int, int, int, int]] = []
|
||||
self._ropes_pending: list[dict[str, Any]] = []
|
||||
self._ropes_metadata: dict[str, dict[str, Any]] = {}
|
||||
self._cfg_companion_queue: deque[tuple[tuple[int, int, int, int], int]] = deque()
|
||||
|
||||
# Per-request position offset for decode after img2img prefill.
|
||||
# Prefill rewrites positions (VAE→0, ViT→1, text→2..N) but the model
|
||||
# runner assigns decode positions starting from prefill_len, not N+1.
|
||||
# offset = rope - prefill_len (a negative number).
|
||||
self._pending_decode_offsets: list[int] = []
|
||||
self._decode_position_offsets: dict[str, int] = {}
|
||||
self._last_img2img_info: tuple[int, int, int, int] | None = None
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -461,7 +453,6 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
self._start_of_image_id = int(_tok.convert_tokens_to_ids("<|vision_start|>"))
|
||||
self._end_of_image_id = int(_tok.convert_tokens_to_ids("<|vision_end|>"))
|
||||
self._img2img_token_id = int(_tok.convert_tokens_to_ids("<|fim_middle|>"))
|
||||
|
||||
self._vae_token_mask: torch.Tensor | None = None
|
||||
self.device = get_local_device()
|
||||
self._install_mot_modules(config)
|
||||
@@ -540,9 +531,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
self._ropes_pending.clear()
|
||||
self._ropes_metadata.clear()
|
||||
self._pending_img2img_info.clear()
|
||||
self._cfg_companion_queue.clear()
|
||||
self._pending_decode_offsets.clear()
|
||||
self._decode_position_offsets.clear()
|
||||
self._last_img2img_info = None
|
||||
self._vae_token_mask = None
|
||||
|
||||
def get_kv_transfer_metadata(
|
||||
@@ -554,12 +543,10 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
meta = self._ropes_metadata.pop(req_id, None)
|
||||
if meta is None:
|
||||
return None
|
||||
# In think-mode img2img the prefill rope doesn't account for decoded
|
||||
# thinking tokens; correct it to num_computed_tokens + offset.
|
||||
# Skip correction when num_computed_tokens is unavailable (None).
|
||||
offset = self._decode_position_offsets.pop(req_id, 0)
|
||||
if offset != 0 and "ropes" in meta and num_computed_tokens is not None:
|
||||
meta["ropes"] = [num_computed_tokens + offset]
|
||||
if num_computed_tokens is not None and "image_shape" in meta:
|
||||
prefill_rope = meta["ropes"][0] if meta.get("ropes") else 0
|
||||
if num_computed_tokens > prefill_rope:
|
||||
meta["ropes"] = [num_computed_tokens]
|
||||
return meta
|
||||
|
||||
def prepare_runner_inputs(
|
||||
@@ -572,48 +559,29 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
num_scheduled_tokens: list[int],
|
||||
input_ids_buffer: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
"""Model-runner hook: adjust inputs before ``forward()``.
|
||||
|
||||
Returns ``(input_ids, positions)`` — possibly modified.
|
||||
|
||||
Two adjustments for BAGEL img2img:
|
||||
|
||||
1. **Restore input_ids** when ``inputs_embeds`` is present so that
|
||||
``_adjust_positions_for_img2img`` can locate the
|
||||
``<|fim_middle|>`` placeholder.
|
||||
2. **Decode position offset**: prefill rewrites positions to a
|
||||
compact scheme (rope ≪ prefill_len). The runner assigns decode
|
||||
positions from ``num_computed_tokens``, which is far too large;
|
||||
apply the stored per-request offset.
|
||||
"""
|
||||
"""Restore input_ids so _adjust_positions_for_img2img can locate
|
||||
the <|fim_middle|> placeholder for thinking-mode pre_text_len
|
||||
detection."""
|
||||
if inputs_embeds is not None and input_ids is None and input_ids_buffer is not None:
|
||||
input_ids = input_ids_buffer
|
||||
|
||||
if self._decode_position_offsets and positions is not None:
|
||||
token_start = 0
|
||||
for i, rid in enumerate(req_ids):
|
||||
sched = num_scheduled_tokens[i]
|
||||
offset = self._decode_position_offsets.get(rid, 0)
|
||||
if offset != 0 and num_computed_tokens[i] > 0:
|
||||
positions[token_start : token_start + sched] += offset
|
||||
token_start += sched
|
||||
|
||||
return input_ids, positions
|
||||
|
||||
def flush_pending_metadata(self, req_ids: list[str]) -> None:
|
||||
"""Map pending metadata (batch order) to req_ids after forward()."""
|
||||
"""Map pending metadata (batch order) to req_ids after forward().
|
||||
|
||||
Guard: if a request already has metadata with ``image_shape``
|
||||
(written during img2img prefill), don't overwrite it with
|
||||
decode-step metadata that lacks ``image_shape``.
|
||||
"""
|
||||
pending = self._ropes_pending
|
||||
self._ropes_pending = []
|
||||
for i, meta in enumerate(pending):
|
||||
if i < len(req_ids):
|
||||
if req_ids[i] not in self._ropes_metadata:
|
||||
self._ropes_metadata[req_ids[i]] = meta
|
||||
|
||||
pending_offsets = self._pending_decode_offsets
|
||||
self._pending_decode_offsets = []
|
||||
for i, offset in enumerate(pending_offsets):
|
||||
if i < len(req_ids) and offset != 0:
|
||||
self._decode_position_offsets[req_ids[i]] = offset
|
||||
rid = req_ids[i]
|
||||
existing = self._ropes_metadata.get(rid)
|
||||
if existing and "image_shape" in existing and "image_shape" not in meta:
|
||||
continue
|
||||
self._ropes_metadata[rid] = meta
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
@@ -727,16 +695,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
num_vit = vit_emb.shape[0] + 2
|
||||
info = (num_vae, num_vit, int(H), int(W))
|
||||
self._pending_img2img_info.append(info)
|
||||
# Only the gen (main) request should add a companion queue entry.
|
||||
# Companion requests (cfg_text, cfg_img) also call this method with
|
||||
# the same image, so guard by checking whether this exact info
|
||||
# tuple is already enqueued. For batched img2img with multiple
|
||||
# concurrent gen requests this correctly adds one entry per unique
|
||||
# image; images with identical (num_vae, num_vit, H, W) that arrive
|
||||
# in the same batch are indistinguishable here and will share one
|
||||
# entry, but that is an uncommon edge case.
|
||||
if not any(entry[0] == info for entry in self._cfg_companion_queue):
|
||||
self._cfg_companion_queue.append((info, 2)) # cfg_text + cfg_img
|
||||
self._last_img2img_info = info
|
||||
|
||||
return tuple(results)
|
||||
|
||||
@@ -755,31 +714,18 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
positions = self._adjust_positions_for_img2img(positions, input_ids)
|
||||
use_mot = True
|
||||
|
||||
elif self._cfg_companion_queue:
|
||||
# Guard: if this looks like a pure decode step (small token count,
|
||||
# no multimodal embeddings), the queue has stale entries from a
|
||||
# previous prefill cycle — clear them instead of consuming.
|
||||
if inputs_embeds is None and seq_len <= 2:
|
||||
self._cfg_companion_queue.clear()
|
||||
elif self._last_img2img_info is not None:
|
||||
info = self._last_img2img_info
|
||||
num_vae, num_vit, _, _ = info
|
||||
num_img2img = num_vae + 1 + num_vit
|
||||
|
||||
if seq_len >= num_img2img:
|
||||
self._pending_img2img_info = [info]
|
||||
positions = self._adjust_positions_for_img2img(positions, input_ids)
|
||||
use_mot = True
|
||||
else:
|
||||
cached, remaining = self._cfg_companion_queue[0]
|
||||
remaining -= 1
|
||||
num_vae, num_vit, img_H, img_W = cached
|
||||
num_img2img = num_vae + 1 + num_vit # +1 separator
|
||||
seq_len = inputs_embeds.shape[0] if inputs_embeds is not None else positions.shape[0]
|
||||
|
||||
if inputs_embeds is not None and seq_len >= num_img2img:
|
||||
self._pending_img2img_info = [cached]
|
||||
positions = self._adjust_positions_for_img2img(positions, input_ids)
|
||||
use_mot = True
|
||||
else:
|
||||
rope = int(positions[seq_len - 1].item()) + 1
|
||||
self._ropes_pending.append({"ropes": [rope]})
|
||||
|
||||
if remaining == 0:
|
||||
self._cfg_companion_queue.popleft()
|
||||
else:
|
||||
self._cfg_companion_queue[0] = (cached, remaining)
|
||||
rope = int(positions[seq_len - 1].item()) + 1
|
||||
self._ropes_pending.append({"ropes": [rope]})
|
||||
|
||||
if use_mot:
|
||||
return self._mot_forward(input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs)
|
||||
@@ -790,27 +736,18 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Rewrite position IDs to match the original BAGEL position scheme:
|
||||
"""Rewrite position IDs for img2img.
|
||||
|
||||
If there are ``pre_text_len`` text tokens before the img2img block::
|
||||
Supports an optional ``pre_text_len`` prefix (thinking-mode) detected
|
||||
via the ``<|fim_middle|>`` token in *input_ids*:
|
||||
|
||||
pre_text → 0, 1, ..., M-1
|
||||
VAE → M (all share)
|
||||
separator→ M
|
||||
ViT → M+1 (all share)
|
||||
post_text→ M+2, M+3, ...
|
||||
pre_text -> 0 .. M-1
|
||||
VAE -> M (all share)
|
||||
separator-> M
|
||||
ViT -> M+1 (all share)
|
||||
post_text-> M+2, M+3, ...
|
||||
|
||||
When no text precedes the img2img block (M=0), this reduces to the
|
||||
simpler scheme: VAE→0, ViT→1, text→2, 3, ...
|
||||
|
||||
Also computes ``self._vae_token_mask`` (bool tensor, True for actual
|
||||
VAE latent patches that should use gen-mode weights) and pushes
|
||||
per-request ropes + image_shape to the FIFO consumed by
|
||||
``get_kv_transfer_metadata``.
|
||||
|
||||
For img2img requests, also stores a decode position offset so that
|
||||
subsequent autoregressive decode steps use positions that continue
|
||||
from the rewritten scheme rather than from the original prefill length.
|
||||
When M=0 (standard img2img) this reduces to VAE->0, ViT->1, text->2..
|
||||
"""
|
||||
info_list = self._pending_img2img_info
|
||||
self._pending_img2img_info = []
|
||||
@@ -836,70 +773,64 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
req_len = end - start
|
||||
|
||||
if img2img_idx < len(info_list):
|
||||
num_vae, num_vit, img_H, img_W = info_list[img2img_idx]
|
||||
cur_info = info_list[img2img_idx]
|
||||
elif self._last_img2img_info is not None:
|
||||
cur_info = self._last_img2img_info
|
||||
else:
|
||||
cur_info = None
|
||||
|
||||
if cur_info is not None:
|
||||
num_vae, num_vit, img_H, img_W = cur_info
|
||||
num_img2img = num_vae + 1 + num_vit # +1 separator
|
||||
|
||||
if req_len >= num_img2img:
|
||||
# Detect offset of img2img tokens within this request
|
||||
# by searching for the img2img placeholder token ID.
|
||||
pre_text_len = 0
|
||||
if input_ids is not None:
|
||||
req_ids = input_ids[start:end]
|
||||
mask = req_ids == self._img2img_token_id
|
||||
indices = mask.nonzero(as_tuple=True)[0]
|
||||
req_ids_slice = input_ids[start:end]
|
||||
indices = (req_ids_slice == self._img2img_token_id).nonzero(as_tuple=True)[0]
|
||||
if indices.numel() > 0:
|
||||
pre_text_len = int(indices[0].item())
|
||||
|
||||
img_start = start + pre_text_len
|
||||
M = pre_text_len
|
||||
img_start = start + M
|
||||
post_text_start = img_start + num_img2img
|
||||
# pre_text_pos: position base for image tokens
|
||||
pre_text_pos = pre_text_len
|
||||
|
||||
# Pre-image text: sequential positions 0..pre_text_pos-1
|
||||
if pre_text_len > 0:
|
||||
if M > 0:
|
||||
new_positions[start:img_start] = torch.arange(
|
||||
0, pre_text_pos, device=positions.device, dtype=positions.dtype
|
||||
0, M, device=positions.device, dtype=positions.dtype
|
||||
)
|
||||
|
||||
# VAE tokens: all share position pre_text_pos
|
||||
new_positions[img_start : img_start + num_vae] = pre_text_pos
|
||||
# Separator: position pre_text_pos
|
||||
new_positions[img_start + num_vae] = pre_text_pos
|
||||
# ViT tokens: all share position pre_text_pos+1
|
||||
new_positions[img_start : img_start + num_vae] = M
|
||||
new_positions[img_start + num_vae] = M # separator
|
||||
vit_start = img_start + num_vae + 1
|
||||
new_positions[vit_start : vit_start + num_vit] = pre_text_pos + 1
|
||||
new_positions[vit_start : vit_start + num_vit] = M + 1
|
||||
|
||||
# Post-image text: sequential positions pre_text_pos+2, pre_text_pos+3, ...
|
||||
num_post_text = end - post_text_start
|
||||
if num_post_text > 0:
|
||||
new_positions[post_text_start:end] = torch.arange(
|
||||
pre_text_pos + 2,
|
||||
pre_text_pos + 2 + num_post_text,
|
||||
M + 2,
|
||||
M + 2 + num_post_text,
|
||||
device=positions.device,
|
||||
dtype=positions.dtype,
|
||||
)
|
||||
|
||||
# VAE gen-mode mask: only actual VAE latent patches (not markers)
|
||||
vae_patches_start = img_start + 1 # skip start_marker
|
||||
vae_patches_end = img_start + num_vae - 1 # before end_marker
|
||||
vae_patches_start = img_start + 1
|
||||
vae_patches_end = img_start + num_vae - 1
|
||||
if vae_patches_end > vae_patches_start:
|
||||
vae_mask[vae_patches_start:vae_patches_end] = True
|
||||
|
||||
rope = pre_text_pos + 2 + num_post_text
|
||||
rope = M + 2 + num_post_text
|
||||
self._ropes_pending.append(
|
||||
{
|
||||
"ropes": [rope],
|
||||
"image_shape": [img_H, img_W],
|
||||
}
|
||||
)
|
||||
decode_offset = rope - req_len
|
||||
self._pending_decode_offsets.append(decode_offset)
|
||||
img2img_idx += 1
|
||||
continue
|
||||
|
||||
rope = int(new_positions[end - 1].item()) + 1
|
||||
self._ropes_pending.append({"ropes": [rope]})
|
||||
self._pending_decode_offsets.append(0)
|
||||
|
||||
self._vae_token_mask = vae_mask if vae_mask.any() else None
|
||||
return new_positions
|
||||
|
||||
@@ -149,7 +149,15 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
encoder_cache=self.encoder_cache,
|
||||
) as ec_connector_output:
|
||||
self._execute_mm_encoder(scheduler_output)
|
||||
return make_empty_encoder_model_runner_output(scheduler_output)
|
||||
|
||||
kv_ids = self.kv_extracted_req_ids
|
||||
self.kv_extracted_req_ids = None
|
||||
|
||||
output = make_empty_encoder_model_runner_output(scheduler_output)
|
||||
if kv_ids:
|
||||
output = copy(output)
|
||||
output.kv_extracted_req_ids = kv_ids
|
||||
return output
|
||||
|
||||
if not num_scheduled_tokens:
|
||||
if (
|
||||
@@ -163,10 +171,20 @@ class NPUARModelRunner(OmniNPUModelRunner):
|
||||
# dummy run to ensure coordinate_batch_across_dp
|
||||
# is called into to avoid out of sync issues.
|
||||
self._dummy_run(1)
|
||||
|
||||
kv_ids = self.kv_extracted_req_ids
|
||||
self.kv_extracted_req_ids = None
|
||||
|
||||
if not has_kv_transfer_group():
|
||||
# Return empty ModelRunnerOutput if no work to do.
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
|
||||
output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
else:
|
||||
output = self.kv_connector_no_forward(scheduler_output, self.vllm_config)
|
||||
|
||||
if kv_ids:
|
||||
output = copy(output)
|
||||
output.kv_extracted_req_ids = kv_ids
|
||||
|
||||
return output
|
||||
if self.cache_config.kv_sharing_fast_prefill:
|
||||
assert not self.num_prompt_logprobs, (
|
||||
"--kv-sharing-fast-prefill produces incorrect "
|
||||
|
||||
@@ -205,24 +205,39 @@ class GPUARModelRunner(OmniGPUModelRunner):
|
||||
encoder_cache=self.encoder_cache,
|
||||
) as ec_connector_output:
|
||||
self._execute_mm_encoder(scheduler_output)
|
||||
return make_empty_encoder_model_runner_output(scheduler_output)
|
||||
|
||||
kv_ids = self.kv_extracted_req_ids
|
||||
self.kv_extracted_req_ids = None
|
||||
|
||||
output = make_empty_encoder_model_runner_output(scheduler_output)
|
||||
if kv_ids:
|
||||
output = copy(output)
|
||||
output.kv_extracted_req_ids = kv_ids
|
||||
return output
|
||||
|
||||
if not num_scheduled_tokens:
|
||||
if (
|
||||
self.parallel_config.distributed_executor_backend == "external_launcher"
|
||||
and self.parallel_config.data_parallel_size > 1
|
||||
):
|
||||
# this is a corner case when both external launcher
|
||||
# and DP are enabled, num_scheduled_tokens could be
|
||||
# 0, and has_unfinished_requests in the outer loop
|
||||
# returns True. before returning early here we call
|
||||
# dummy run to ensure coordinate_batch_across_dp
|
||||
# is called into to avoid out of sync issues.
|
||||
self._dummy_run(1)
|
||||
|
||||
# Capture KV extraction results before early return;
|
||||
# sample_tokens() is skipped on this path so the IDs
|
||||
# would otherwise be silently overwritten next step.
|
||||
kv_ids = self.kv_extracted_req_ids
|
||||
self.kv_extracted_req_ids = None
|
||||
|
||||
if not has_kv_transfer_group():
|
||||
# Return empty ModelRunnerOutput if no work to do.
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
|
||||
output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
else:
|
||||
output = self.kv_connector_no_forward(scheduler_output, self.vllm_config)
|
||||
|
||||
if kv_ids:
|
||||
output = copy(output)
|
||||
output.kv_extracted_req_ids = kv_ids
|
||||
|
||||
return output
|
||||
|
||||
if self.cache_config.kv_sharing_fast_prefill:
|
||||
assert not self.num_prompt_logprobs, (
|
||||
|
||||
Reference in new issue
Block a user