[Core] Support Async & Sync AutoRegressive Scheduling (#3306)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
703e31fc47
commit
bb239fa949
6 files changed
+87
-77
No files matched your search
@@ -562,6 +562,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
*,
|
||||
num_computed_tokens: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
# NOTE: num_computed_tokens will not include async placeholders
|
||||
meta = self._ropes_metadata.pop(req_id, None)
|
||||
if meta is None:
|
||||
return None
|
||||
@@ -854,7 +855,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
{
|
||||
"ropes": [rope],
|
||||
"image_shape": [img_H, img_W],
|
||||
"prefill_position_count": int(end - start),
|
||||
"prefill_position_count": req_len,
|
||||
}
|
||||
)
|
||||
img2img_idx += 1
|
||||
|
||||
Reference in new issue
Block a user