fix conflicts
This commit is contained in:
1 parent
22f36de34e
commit
171f3c1ea7
42 files changed
+2930
-1842
No files matched your search
@@ -1,21 +1,21 @@
|
||||
from typing import Dict, Callable, Optional, Any, Union
|
||||
from typing import Any, Callable, Dict, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessorOutput, RequestState, RequestOutputCollector
|
||||
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
|
||||
from vllm.v1.engine.output_processor import (
|
||||
OutputProcessorOutput,
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import EngineCoreOutput
|
||||
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -67,8 +67,9 @@ class OmniRequestState(RequestState):
|
||||
request_id=request.request_id,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
lora_name=(request.lora_request.name
|
||||
if request.lora_request is not None else None),
|
||||
lora_name=(
|
||||
request.lora_request.name if request.lora_request is not None else None
|
||||
),
|
||||
output_kind=output_kind,
|
||||
prompt=prompt,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
@@ -80,8 +81,9 @@ class OmniRequestState(RequestState):
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
def add_multimodal_tensor(self, tensor: Optional[torch.Tensor],
|
||||
mm_type: Optional[str]) -> None:
|
||||
def add_multimodal_tensor(
|
||||
self, tensor: Optional[torch.Tensor], mm_type: Optional[str]
|
||||
) -> None:
|
||||
if tensor is None:
|
||||
return
|
||||
try:
|
||||
@@ -126,36 +128,41 @@ class OmniRequestState(RequestState):
|
||||
pass
|
||||
|
||||
request_id = self.request_id
|
||||
output = self._new_completion_output(new_token_ids, finish_reason,
|
||||
stop_reason)
|
||||
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
|
||||
|
||||
if self.parent_req is None:
|
||||
outputs = [output]
|
||||
else:
|
||||
request_id, outputs, finished = self.parent_req.get_outputs(
|
||||
request_id, output)
|
||||
request_id, output
|
||||
)
|
||||
if not outputs:
|
||||
return None
|
||||
|
||||
return self._new_request_output(request_id, outputs, finished,
|
||||
kv_transfer_params)
|
||||
return self._new_request_output(
|
||||
request_id, outputs, finished, kv_transfer_params
|
||||
)
|
||||
|
||||
def _new_completion_output(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Optional[Union[int, str]]
|
||||
stop_reason: Optional[Union[int, str]],
|
||||
) -> Any:
|
||||
# Reuse base text/logprobs logic, then annotate with pooling_result.
|
||||
base_output = super()._new_completion_output(token_ids, finish_reason,
|
||||
stop_reason)
|
||||
base_output = super()._new_completion_output(
|
||||
token_ids, finish_reason, stop_reason
|
||||
)
|
||||
try:
|
||||
if self.mm_accumulated is not None:
|
||||
tensor = self.mm_accumulated
|
||||
try:
|
||||
tensor = tensor.detach().to("cpu")
|
||||
except Exception:
|
||||
logger.debug("Failed to move accumulated multimodal tensor to CPU", exc_info=True)
|
||||
logger.debug(
|
||||
"Failed to move accumulated multimodal tensor to CPU",
|
||||
exc_info=True,
|
||||
)
|
||||
# Attach on the completion output for downstream consumers.
|
||||
if not hasattr(base_output, "multimodal_output"):
|
||||
setattr(base_output, "multimodal_output", {})
|
||||
@@ -177,17 +184,24 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
produce the correct RequestOutput/PoolingRequestOutput.
|
||||
- Allow custom per-modality handlers via register_handler().
|
||||
"""
|
||||
def __init__(self, tokenizer: TokenizerGroup, log_stats: bool, engine_core_output_type: Optional[str] = None):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerGroup,
|
||||
log_stats: bool,
|
||||
engine_core_output_type: Optional[str] = None,
|
||||
):
|
||||
super().__init__(tokenizer=tokenizer, log_stats=log_stats)
|
||||
self.output_handlers: Dict[str, Callable[[EngineCoreOutput], None]] = {}
|
||||
self._reqid_to_mm_type: Dict[str, str] = {}
|
||||
self.request_states: dict[str, OmniRequestState] = {}
|
||||
self.engine_core_output_type = engine_core_output_type
|
||||
|
||||
def register_handler(self, modality: str,
|
||||
handler: Callable[[EngineCoreOutput], None]) -> None:
|
||||
|
||||
def register_handler(
|
||||
self, modality: str, handler: Callable[[EngineCoreOutput], None]
|
||||
) -> None:
|
||||
self.output_handlers[modality.lower()] = handler
|
||||
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
request: EngineCoreRequest,
|
||||
@@ -200,21 +214,26 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
if request_id in self.request_states:
|
||||
raise ValueError(f"Request id {request_id} already running.")
|
||||
|
||||
tokenizer = None if not self.tokenizer else \
|
||||
self.tokenizer.get_lora_tokenizer(request.lora_request)
|
||||
tokenizer = (
|
||||
None
|
||||
if not self.tokenizer
|
||||
else self.tokenizer.get_lora_tokenizer(request.lora_request)
|
||||
)
|
||||
|
||||
req_state = OmniRequestState.from_new_request(tokenizer=tokenizer,
|
||||
request=request,
|
||||
prompt=prompt,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
queue=queue,
|
||||
log_stats=self.log_stats)
|
||||
req_state = OmniRequestState.from_new_request(
|
||||
tokenizer=tokenizer,
|
||||
request=request,
|
||||
prompt=prompt,
|
||||
parent_req=parent_req,
|
||||
request_index=request_index,
|
||||
queue=queue,
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
self.request_states[request_id] = req_state
|
||||
self.lora_states.add_request(req_state)
|
||||
if parent_req:
|
||||
self.parent_requests[parent_req.request_id] = parent_req
|
||||
|
||||
|
||||
def process_outputs(
|
||||
self,
|
||||
engine_core_outputs: list[EngineCoreOutput],
|
||||
@@ -238,9 +257,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
continue
|
||||
|
||||
# 1) Stats
|
||||
self._update_stats_from_output(req_state, eco,
|
||||
engine_core_timestamp,
|
||||
iteration_stats)
|
||||
self._update_stats_from_output(
|
||||
req_state, eco, engine_core_timestamp, iteration_stats
|
||||
)
|
||||
|
||||
new_token_ids = eco.new_token_ids
|
||||
pooling_output = eco.pooling_output
|
||||
@@ -254,7 +273,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
stop_string = req_state.detokenizer.update(
|
||||
new_token_ids, finish_reason == FinishReason.STOP)
|
||||
new_token_ids, finish_reason == FinishReason.STOP
|
||||
)
|
||||
if stop_string:
|
||||
finish_reason = FinishReason.STOP
|
||||
stop_reason = stop_string
|
||||
@@ -262,11 +282,19 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
|
||||
# 2.5) Accumulate multimodal tensors in RequestState
|
||||
try:
|
||||
mm_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
|
||||
if pooling_output is not None and isinstance(req_state, OmniRequestState):
|
||||
mm_type = (
|
||||
getattr(eco, "output_type", self.engine_core_output_type) or ""
|
||||
).lower()
|
||||
if pooling_output is not None and isinstance(
|
||||
req_state, OmniRequestState
|
||||
):
|
||||
req_state.add_multimodal_tensor(pooling_output, mm_type)
|
||||
except Exception:
|
||||
logger.debug("Failed to accumulate multimodal tensor for request %s", req_id, exc_info=True)
|
||||
logger.debug(
|
||||
"Failed to accumulate multimodal tensor for request %s",
|
||||
req_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# 3) Create RequestOutput objects, forcing combined mode to keep ids
|
||||
pooling_for_make = pooling_output
|
||||
@@ -274,20 +302,29 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
# Do not consume pooling path now; keep ids and attach mm later
|
||||
pooling_for_make = None
|
||||
|
||||
ro = req_state.make_request_output(new_token_ids, pooling_for_make,
|
||||
finish_reason, stop_reason,
|
||||
kv_transfer_params,
|
||||
num_cached_tokens)
|
||||
ro = req_state.make_request_output(
|
||||
new_token_ids,
|
||||
pooling_for_make,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
num_cached_tokens,
|
||||
)
|
||||
if ro:
|
||||
# Attach accumulated multimodal payload if any
|
||||
try:
|
||||
if isinstance(req_state, OmniRequestState) and req_state.mm_accumulated is not None:
|
||||
if (
|
||||
isinstance(req_state, OmniRequestState)
|
||||
and req_state.mm_accumulated is not None
|
||||
):
|
||||
mm_key = req_state.mm_type or "latents"
|
||||
if not hasattr(ro, "multimodal_output"):
|
||||
setattr(ro, "multimodal_output", {})
|
||||
ro.multimodal_output[mm_key] = req_state.mm_accumulated
|
||||
except Exception:
|
||||
logger.exception("Error attaching multimodal payload in process_outputs")
|
||||
logger.exception(
|
||||
"Error attaching multimodal payload in process_outputs"
|
||||
)
|
||||
if req_state.queue is not None:
|
||||
req_state.queue.put(ro)
|
||||
else:
|
||||
@@ -301,8 +338,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
self.parent_requests.pop(parent_req.request_id, None)
|
||||
if not eco.finished:
|
||||
reqs_to_abort.append(req_id)
|
||||
self._update_stats_from_finished(req_state, finish_reason,
|
||||
iteration_stats)
|
||||
self._update_stats_from_finished(
|
||||
req_state, finish_reason, iteration_stats
|
||||
)
|
||||
# Cleanup per-request mm state
|
||||
if isinstance(req_state, OmniRequestState):
|
||||
req_state.mm_accumulated = None
|
||||
@@ -312,10 +350,12 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
request_outputs=request_outputs,
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
)
|
||||
|
||||
|
||||
# ---- routing helpers ----
|
||||
def _route_and_normalize(self, eco: EngineCoreOutput) -> None:
|
||||
output_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
|
||||
output_type = (
|
||||
getattr(eco, "output_type", self.engine_core_output_type) or ""
|
||||
).lower()
|
||||
|
||||
# Custom handler first (if registered)
|
||||
if output_type in self.output_handlers:
|
||||
@@ -347,7 +387,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
"""Ensure image tensors are surfaced via pooling_output for vLLM."""
|
||||
if eco.pooling_output is None:
|
||||
tensor = self._extract_from_multimodal_outputs(
|
||||
eco, keys=("image", "images", "pixel_values", "pixels"))
|
||||
eco, keys=("image", "images", "pixel_values", "pixels")
|
||||
)
|
||||
if tensor is not None:
|
||||
eco.pooling_output = tensor
|
||||
|
||||
@@ -357,8 +398,17 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
# Preserve text tokens as-is; ensure pooling_output carries image/latents
|
||||
if eco.pooling_output is None:
|
||||
tensor = self._extract_from_multimodal_outputs(
|
||||
eco, keys=("image", "images", "pixel_values", "pixels",
|
||||
"latent", "latents", "z"))
|
||||
eco,
|
||||
keys=(
|
||||
"image",
|
||||
"images",
|
||||
"pixel_values",
|
||||
"pixels",
|
||||
"latent",
|
||||
"latents",
|
||||
"z",
|
||||
),
|
||||
)
|
||||
if tensor is not None:
|
||||
eco.pooling_output = tensor
|
||||
|
||||
@@ -366,7 +416,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
"""Ensure latent tensors are surfaced via pooling_output."""
|
||||
if eco.pooling_output is None:
|
||||
tensor = self._extract_from_multimodal_outputs(
|
||||
eco, keys=("latent", "latents", "z", "posterior"))
|
||||
eco, keys=("latent", "latents", "z", "posterior")
|
||||
)
|
||||
if tensor is not None:
|
||||
eco.pooling_output = tensor
|
||||
|
||||
@@ -374,8 +425,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
"""Ensure audio tensors are surfaced via pooling_output."""
|
||||
if eco.pooling_output is None:
|
||||
tensor = self._extract_from_multimodal_outputs(
|
||||
eco, keys=("audio", "audios", "wav", "waveform",
|
||||
"audio_pcm", "pcm"))
|
||||
eco, keys=("audio", "audios", "wav", "waveform", "audio_pcm", "pcm")
|
||||
)
|
||||
if tensor is not None:
|
||||
eco.pooling_output = tensor
|
||||
|
||||
@@ -408,4 +459,4 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
|
||||
for v in mm.values():
|
||||
if isinstance(v, torch.Tensor):
|
||||
return v
|
||||
return None
|
||||
return None
|
||||
Reference in new issue
Block a user