anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Dev/rebase v0.15.0 (#1159)

Browse Source 
Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk>
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
Zhou Taichangand汪志鹏 authored and GitHub committed 2026-02-03 12:24:45 +08:00
1 parent 9494d691bf
commit 70ef548448
44 files changed
+648 -554
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vllm_omni/engine/output_processor.py
+44 -112
View File
@@ -2,6 +2,7 @@ from ast import Dict
from collections.abc import Callable
from typing import Any
import numpy as np
import torch
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
@@ -9,7 +10,11 @@ from vllm.sampling_params import RequestOutputKind
from vllm.tokenizers import TokenizerLike
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
from vllm.v1.engine.output_processor import OutputProcessorOutput, RequestOutputCollector, RequestState
from vllm.v1.engine.output_processor import (
OutputProcessorOutput,
RequestOutputCollector,
RequestState,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.stats import IterationStats
@@ -140,6 +145,7 @@ class OmniRequestState(RequestState):
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
routed_experts: np.ndarray | None = None,
) -> OmniRequestOutput | PoolingRequestOutput | None:
"""Create a request output from generation results.
@@ -158,12 +164,27 @@ class OmniRequestState(RequestState):
OmniRequestOutput or PoolingRequestOutput if output should be
emitted (based on finish status and output kind), None otherwise
"""
# Pooling-only requests should follow base behavior.
if self.detokenizer is None and pooling_output is not None:
return super().make_request_output(
new_token_ids,
pooling_output,
finish_reason,
stop_reason,
kv_transfer_params,
routed_experts,
)
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
if not finished and final_only:
return None
# Consolidate accumulated tensors when finishing.
if finished:
self._consolidate_multimodal_tensors()
if self.stream_interval > 1:
assert self.detokenizer is not None
@@ -185,7 +206,7 @@ class OmniRequestState(RequestState):
self.sent_tokens_offset = len(self.detokenizer.output_token_ids)
request_id = self.request_id
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason, routed_experts)
if self.parent_req is None:
outputs = [output]
@@ -201,9 +222,10 @@ class OmniRequestState(RequestState):
token_ids: list[int],
finish_reason: FinishReason | None,
stop_reason: int | str | None,
routed_experts: np.ndarray | None = None,
) -> Any:
# Reuse base text/logprobs logic, then annotate with pooling_result.
base_output = super()._new_completion_output(token_ids, finish_reason, stop_reason)
base_output = super()._new_completion_output(token_ids, finish_reason, stop_reason, routed_experts)
try:
if self.mm_accumulated is not None:
# Attach accumulated multimodal dict on the completion output
@@ -251,7 +273,6 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
super().__init__(tokenizer=tokenizer, log_stats=log_stats)
self.output_handlers: dict[str, Callable[[EngineCoreOutput], None]] = {}
self._reqid_to_mm_type: dict[str, str] = {}
self.request_states: dict[str, OmniRequestState] = {}
self.engine_core_output_type = engine_core_output_type
def register_handler(self, modality: str, handler: Callable[[EngineCoreOutput], None]) -> None:
@@ -291,8 +312,10 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
ValueError: If the request ID is already registered
"""
request_id = request.request_id
if request_id in self.request_states:
raise ValueError(f"Request id {request_id} already running.")
req_state = self.request_states.get(request_id)
if req_state is not None:
self._update_streaming_request_state(req_state, request, prompt)
return
req_state = OmniRequestState.from_new_request(
tokenizer=self.tokenizer,
@@ -304,9 +327,12 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
log_stats=self.log_stats,
stream_interval=self.stream_interval,
)
if self._requests_drained.is_set():
self._requests_drained.clear()
self.request_states[request_id] = req_state
if parent_req:
self.parent_requests[parent_req.request_id] = parent_req
self.external_req_ids[req_state.external_req_id].append(request_id)
def process_outputs(
self,
@@ -314,121 +340,27 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
engine_core_timestamp: float | None = None,
iteration_stats: IterationStats | None = None,
) -> OutputProcessorOutput:
"""Process engine core outputs into request outputs.
Converts EngineCoreOutput objects into RequestOutput objects,
handling multimodal outputs by routing them through appropriate
processors and accumulating tensors in request states.
Args:
engine_core_outputs: List of engine core outputs to process
engine_core_timestamp: Optional timestamp for the outputs
iteration_stats: Optional iteration statistics
Returns:
OutputProcessorOutput containing processed request outputs
and list of request IDs to abort
"""
self._reqid_to_mm_type.clear()
for eco in engine_core_outputs:
mm_type = (self.engine_core_output_type or "").lower()
if mm_type:
self._reqid_to_mm_type[eco.request_id] = mm_type
self._route_and_normalize(eco)
# Build RequestOutputs without delegating to base, so we can keep ids
request_outputs: list[Any] = []
reqs_to_abort: list[str] = []
for eco in engine_core_outputs:
req_id = eco.request_id
req_state = self.request_states.get(req_id)
if req_state is None:
req_state = self.request_states.get(eco.request_id)
if req_state is None or not isinstance(req_state, OmniRequestState):
continue
# 1) Stats
self._update_stats_from_output(req_state, eco, engine_core_timestamp, iteration_stats)
new_token_ids = eco.new_token_ids
pooling_output = eco.pooling_output
finish_reason = eco.finish_reason
stop_reason = eco.stop_reason
kv_transfer_params = eco.kv_transfer_params
req_state.num_cached_tokens = eco.num_cached_tokens
req_state.is_prefilling = False
# 2) Detokenize and logprobs when text path
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
stop_string = req_state.detokenizer.update(new_token_ids, finish_reason == FinishReason.STOP)
if stop_string:
finish_reason = FinishReason.STOP
stop_reason = stop_string
req_state.logprobs_processor.update_from_output(eco)
# 2.5) Accumulate multimodal tensors in RequestState
try:
mm_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
if pooling_output is not None and isinstance(req_state, OmniRequestState):
req_state.add_multimodal_tensor(pooling_output, mm_type)
except Exception:
logger.debug(
"Failed to accumulate multimodal tensor for request %s",
req_id,
exc_info=True,
if eco.pooling_output is not None and req_state.detokenizer is not None:
req_state.add_multimodal_tensor(
eco.pooling_output,
(getattr(eco, "output_type", self.engine_core_output_type) or "").lower(),
)
# Force text path in base processor for multimodal outputs.
eco.pooling_output = None
# 3) Create RequestOutput objects, forcing combined mode to keep ids
pooling_for_make = pooling_output
if pooling_output is not None and new_token_ids:
# Do not consume pooling path now; keep ids and attach mm later
pooling_for_make = None
# Consolidate any accumulated tensor lists before creating output
if isinstance(req_state, OmniRequestState) and finish_reason is not None:
req_state._consolidate_multimodal_tensors()
ro = req_state.make_request_output(
new_token_ids,
pooling_for_make,
finish_reason,
stop_reason,
kv_transfer_params,
)
if ro:
# Attach accumulated multimodal payload if any
try:
if isinstance(req_state, OmniRequestState) and req_state.mm_accumulated is not None:
if not hasattr(ro, "multimodal_output"):
setattr(ro, "multimodal_output", {})
ro.multimodal_output = req_state.mm_accumulated
except Exception:
logger.exception("Error attaching multimodal payload in process_outputs")
if req_state.queue is not None:
req_state.queue.put(ro)
else:
request_outputs.append(ro)
# 4) Free completed
if finish_reason is not None:
self.request_states.pop(req_id)
parent_req = req_state.parent_req
if parent_req and not parent_req.child_requests:
self.parent_requests.pop(parent_req.request_id, None)
if not self.request_states:
self._requests_drained.set()
if not eco.finished:
reqs_to_abort.append(req_id)
self._update_stats_from_finished(req_state, finish_reason, iteration_stats)
if self.tracer:
self.do_tracing(eco, req_state, iteration_stats)
# Cleanup per-request mm state
if isinstance(req_state, OmniRequestState):
req_state.mm_accumulated = None
req_state.mm_type = None
return OutputProcessorOutput(
request_outputs=request_outputs,
reqs_to_abort=reqs_to_abort,
return super().process_outputs(
engine_core_outputs,
engine_core_timestamp=engine_core_timestamp,
iteration_stats=iteration_stats,
)
# ---- routing helpers ----