fix conflicts

This commit is contained in:
Gaohan123 committed 2025-10-24 16:49:19 +08:00
1 parent 22f36de34e
commit 171f3c1ea7
42 files changed
+2930 -1842

No files matched your search

+114 -63
View File
@@ -1,21 +1,21 @@
from typing import Dict, Callable, Optional, Any, Union
from typing import Any, Callable, Dict, Optional, Union
import torch
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
from vllm.v1.engine.output_processor import OutputProcessorOutput, RequestState, RequestOutputCollector
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
from vllm.v1.engine import FinishReason
from vllm.v1.metrics.stats import IterationStats
from vllm.logger import init_logger
from vllm.sampling_params import RequestOutputKind
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.v1.engine import EngineCoreRequest
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
from vllm.v1.engine.logprobs import LogprobsProcessor
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
from vllm.v1.engine.output_processor import (
OutputProcessorOutput,
RequestOutputCollector,
RequestState,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.logger import init_logger
from vllm.v1.engine import EngineCoreOutput
from vllm.v1.metrics.stats import IterationStats
logger = init_logger(__name__)
@@ -67,8 +67,9 @@ class OmniRequestState(RequestState):
request_id=request.request_id,
parent_req=parent_req,
request_index=request_index,
lora_name=(request.lora_request.name
if request.lora_request is not None else None),
lora_name=(
request.lora_request.name if request.lora_request is not None else None
),
output_kind=output_kind,
prompt=prompt,
prompt_token_ids=request.prompt_token_ids,
@@ -80,8 +81,9 @@ class OmniRequestState(RequestState):
log_stats=log_stats,
)
def add_multimodal_tensor(self, tensor: Optional[torch.Tensor],
mm_type: Optional[str]) -> None:
def add_multimodal_tensor(
self, tensor: Optional[torch.Tensor], mm_type: Optional[str]
) -> None:
if tensor is None:
return
try:
@@ -126,36 +128,41 @@ class OmniRequestState(RequestState):
pass
request_id = self.request_id
output = self._new_completion_output(new_token_ids, finish_reason,
stop_reason)
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
if self.parent_req is None:
outputs = [output]
else:
request_id, outputs, finished = self.parent_req.get_outputs(
request_id, output)
request_id, output
)
if not outputs:
return None
return self._new_request_output(request_id, outputs, finished,
kv_transfer_params)
return self._new_request_output(
request_id, outputs, finished, kv_transfer_params
)
def _new_completion_output(
self,
token_ids: list[int],
finish_reason: Optional[FinishReason],
stop_reason: Optional[Union[int, str]]
stop_reason: Optional[Union[int, str]],
) -> Any:
# Reuse base text/logprobs logic, then annotate with pooling_result.
base_output = super()._new_completion_output(token_ids, finish_reason,
stop_reason)
base_output = super()._new_completion_output(
token_ids, finish_reason, stop_reason
)
try:
if self.mm_accumulated is not None:
tensor = self.mm_accumulated
try:
tensor = tensor.detach().to("cpu")
except Exception:
logger.debug("Failed to move accumulated multimodal tensor to CPU", exc_info=True)
logger.debug(
"Failed to move accumulated multimodal tensor to CPU",
exc_info=True,
)
# Attach on the completion output for downstream consumers.
if not hasattr(base_output, "multimodal_output"):
setattr(base_output, "multimodal_output", {})
@@ -177,17 +184,24 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
produce the correct RequestOutput/PoolingRequestOutput.
- Allow custom per-modality handlers via register_handler().
"""
def __init__(self, tokenizer: TokenizerGroup, log_stats: bool, engine_core_output_type: Optional[str] = None):
def __init__(
self,
tokenizer: TokenizerGroup,
log_stats: bool,
engine_core_output_type: Optional[str] = None,
):
super().__init__(tokenizer=tokenizer, log_stats=log_stats)
self.output_handlers: Dict[str, Callable[[EngineCoreOutput], None]] = {}
self._reqid_to_mm_type: Dict[str, str] = {}
self.request_states: dict[str, OmniRequestState] = {}
self.engine_core_output_type = engine_core_output_type
def register_handler(self, modality: str,
handler: Callable[[EngineCoreOutput], None]) -> None:
def register_handler(
self, modality: str, handler: Callable[[EngineCoreOutput], None]
) -> None:
self.output_handlers[modality.lower()] = handler
def add_request(
self,
request: EngineCoreRequest,
@@ -200,21 +214,26 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
if request_id in self.request_states:
raise ValueError(f"Request id {request_id} already running.")
tokenizer = None if not self.tokenizer else \
self.tokenizer.get_lora_tokenizer(request.lora_request)
tokenizer = (
None
if not self.tokenizer
else self.tokenizer.get_lora_tokenizer(request.lora_request)
)
req_state = OmniRequestState.from_new_request(tokenizer=tokenizer,
request=request,
prompt=prompt,
parent_req=parent_req,
request_index=request_index,
queue=queue,
log_stats=self.log_stats)
req_state = OmniRequestState.from_new_request(
tokenizer=tokenizer,
request=request,
prompt=prompt,
parent_req=parent_req,
request_index=request_index,
queue=queue,
log_stats=self.log_stats,
)
self.request_states[request_id] = req_state
self.lora_states.add_request(req_state)
if parent_req:
self.parent_requests[parent_req.request_id] = parent_req
def process_outputs(
self,
engine_core_outputs: list[EngineCoreOutput],
@@ -238,9 +257,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
continue
# 1) Stats
self._update_stats_from_output(req_state, eco,
engine_core_timestamp,
iteration_stats)
self._update_stats_from_output(
req_state, eco, engine_core_timestamp, iteration_stats
)
new_token_ids = eco.new_token_ids
pooling_output = eco.pooling_output
@@ -254,7 +273,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
stop_string = req_state.detokenizer.update(
new_token_ids, finish_reason == FinishReason.STOP)
new_token_ids, finish_reason == FinishReason.STOP
)
if stop_string:
finish_reason = FinishReason.STOP
stop_reason = stop_string
@@ -262,11 +282,19 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
# 2.5) Accumulate multimodal tensors in RequestState
try:
mm_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
if pooling_output is not None and isinstance(req_state, OmniRequestState):
mm_type = (
getattr(eco, "output_type", self.engine_core_output_type) or ""
).lower()
if pooling_output is not None and isinstance(
req_state, OmniRequestState
):
req_state.add_multimodal_tensor(pooling_output, mm_type)
except Exception:
logger.debug("Failed to accumulate multimodal tensor for request %s", req_id, exc_info=True)
logger.debug(
"Failed to accumulate multimodal tensor for request %s",
req_id,
exc_info=True,
)
# 3) Create RequestOutput objects, forcing combined mode to keep ids
pooling_for_make = pooling_output
@@ -274,20 +302,29 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
# Do not consume pooling path now; keep ids and attach mm later
pooling_for_make = None
ro = req_state.make_request_output(new_token_ids, pooling_for_make,
finish_reason, stop_reason,
kv_transfer_params,
num_cached_tokens)
ro = req_state.make_request_output(
new_token_ids,
pooling_for_make,
finish_reason,
stop_reason,
kv_transfer_params,
num_cached_tokens,
)
if ro:
# Attach accumulated multimodal payload if any
try:
if isinstance(req_state, OmniRequestState) and req_state.mm_accumulated is not None:
if (
isinstance(req_state, OmniRequestState)
and req_state.mm_accumulated is not None
):
mm_key = req_state.mm_type or "latents"
if not hasattr(ro, "multimodal_output"):
setattr(ro, "multimodal_output", {})
ro.multimodal_output[mm_key] = req_state.mm_accumulated
except Exception:
logger.exception("Error attaching multimodal payload in process_outputs")
logger.exception(
"Error attaching multimodal payload in process_outputs"
)
if req_state.queue is not None:
req_state.queue.put(ro)
else:
@@ -301,8 +338,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
self.parent_requests.pop(parent_req.request_id, None)
if not eco.finished:
reqs_to_abort.append(req_id)
self._update_stats_from_finished(req_state, finish_reason,
iteration_stats)
self._update_stats_from_finished(
req_state, finish_reason, iteration_stats
)
# Cleanup per-request mm state
if isinstance(req_state, OmniRequestState):
req_state.mm_accumulated = None
@@ -312,10 +350,12 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
request_outputs=request_outputs,
reqs_to_abort=reqs_to_abort,
)
# ---- routing helpers ----
def _route_and_normalize(self, eco: EngineCoreOutput) -> None:
output_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
output_type = (
getattr(eco, "output_type", self.engine_core_output_type) or ""
).lower()
# Custom handler first (if registered)
if output_type in self.output_handlers:
@@ -347,7 +387,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
"""Ensure image tensors are surfaced via pooling_output for vLLM."""
if eco.pooling_output is None:
tensor = self._extract_from_multimodal_outputs(
eco, keys=("image", "images", "pixel_values", "pixels"))
eco, keys=("image", "images", "pixel_values", "pixels")
)
if tensor is not None:
eco.pooling_output = tensor
@@ -357,8 +398,17 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
# Preserve text tokens as-is; ensure pooling_output carries image/latents
if eco.pooling_output is None:
tensor = self._extract_from_multimodal_outputs(
eco, keys=("image", "images", "pixel_values", "pixels",
"latent", "latents", "z"))
eco,
keys=(
"image",
"images",
"pixel_values",
"pixels",
"latent",
"latents",
"z",
),
)
if tensor is not None:
eco.pooling_output = tensor
@@ -366,7 +416,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
"""Ensure latent tensors are surfaced via pooling_output."""
if eco.pooling_output is None:
tensor = self._extract_from_multimodal_outputs(
eco, keys=("latent", "latents", "z", "posterior"))
eco, keys=("latent", "latents", "z", "posterior")
)
if tensor is not None:
eco.pooling_output = tensor
@@ -374,8 +425,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
"""Ensure audio tensors are surfaced via pooling_output."""
if eco.pooling_output is None:
tensor = self._extract_from_multimodal_outputs(
eco, keys=("audio", "audios", "wav", "waveform",
"audio_pcm", "pcm"))
eco, keys=("audio", "audios", "wav", "waveform", "audio_pcm", "pcm")
)
if tensor is not None:
eco.pooling_output = tensor
@@ -408,4 +459,4 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
for v in mm.values():
if isinstance(v, torch.Tensor):
return v
return None
return None