Shi Boao Bowieandbjf-frz
a432b8a325
Signed-off-by: Boao Shi <aoibosh@connect.hku.hk> Signed-off-by: bjf-frz <frz123db@gmail.com> Co-authored-by: bjf-frz <frz123db@gmail.com>
262 lines
10 KiB
Python
262 lines
10 KiB
Python
"""Multimodal output data structures for vLLM-Omni.
|
|
|
|
This module defines structured types for multimodal outputs.
|
|
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator, Mapping
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
import torch
|
|
from vllm.logger import init_logger
|
|
from vllm.outputs import CompletionOutput
|
|
|
|
from vllm_omni.outputs.output_modality import TensorAccumulationStrategy
|
|
from vllm_omni.outputs.utils import _is_tensor_list, _to_cpu
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# Keys whose values are metadata scalars (e.g. audio sample rate) but may
|
|
# arrive as 0-d torch.Tensors — from_dict routes all tensors into .tensors,
|
|
# so we relocate them to .metadata before consolidation to avoid a bogus
|
|
# torch.cat attempt and its warn-and-keep-last fallback.
|
|
_METADATA_TENSOR_KEYS: frozenset[str] = frozenset({"sr", "sample_rate", "audio_sample_rate"})
|
|
|
|
|
|
def _cat_tensors(
|
|
tensors: list[torch.Tensor],
|
|
strategy: TensorAccumulationStrategy,
|
|
) -> torch.Tensor:
|
|
"""Concatenate a list of tensors according to *strategy*."""
|
|
if strategy == TensorAccumulationStrategy.CONCAT_LAST:
|
|
return torch.cat(tensors, dim=-1)
|
|
if strategy == TensorAccumulationStrategy.REPLACE:
|
|
return tensors[-1]
|
|
# CONCAT_DIM0 / APPEND_LIST / default
|
|
return torch.cat(tensors, dim=0)
|
|
|
|
|
|
def _consolidate_tensor_list(
|
|
key: str,
|
|
tensor_list: list[torch.Tensor],
|
|
strategy: TensorAccumulationStrategy,
|
|
) -> torch.Tensor:
|
|
"""Concatenate a deferred tensor list, with fallbacks on shape mismatch."""
|
|
try:
|
|
return _cat_tensors(tensor_list, strategy)
|
|
except RuntimeError:
|
|
# [TODO: this part is for async chunk, not for audio only. can we come up with a design
|
|
# that identify this by whether using async chunk instead of modality?]
|
|
if key != "audio":
|
|
logger.warning("Error concatenating tensor for key %s; keeping last tensor", key)
|
|
return tensor_list[-1]
|
|
# Audio chunks may have mismatched shapes; retry along the last dim,
|
|
# then fall back to flattening each chunk.
|
|
try:
|
|
return torch.cat(tensor_list, dim=-1)
|
|
except RuntimeError:
|
|
return torch.cat([chunk.reshape(-1) for chunk in tensor_list], dim=0)
|
|
|
|
|
|
def _append_entries(store: dict[str, Any], incoming: dict[str, Any]) -> None:
|
|
"""Append *incoming* entries into *store*.
|
|
|
|
Tensor values are collected into lists (deferred concatenation, avoiding
|
|
O(n²) repeated torch.cat); non-tensor values replace the existing entry.
|
|
"""
|
|
for key, new_value in incoming.items():
|
|
existing = store.get(key)
|
|
if isinstance(existing, list) and isinstance(new_value, torch.Tensor):
|
|
existing.append(new_value)
|
|
elif isinstance(existing, torch.Tensor) and isinstance(new_value, torch.Tensor):
|
|
store[key] = [existing, new_value]
|
|
else:
|
|
store[key] = new_value
|
|
|
|
|
|
@dataclass(eq=False)
|
|
class MultimodalPayload(Mapping):
|
|
"""Structured multimodal output payload.
|
|
|
|
Implements ``collections.abc.Mapping`` so that ``isinstance(payload, dict)``
|
|
style checks in downstream code can be replaced with duck-typing, and
|
|
``payload.get(key)``, ``payload[key]``, ``key in payload``, ``len(payload)``
|
|
all work seamlessly for both tensors and metadata.
|
|
|
|
Attributes:
|
|
tensors: Dictionary mapping modality/key names to their tensors.
|
|
metadata: Optional dictionary for non-tensor metadata
|
|
(e.g., sample rate for audio, image dimensions).
|
|
"""
|
|
|
|
tensors: dict[str, torch.Tensor] = field(default_factory=dict)
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
@property
|
|
def primary_tensor(self) -> torch.Tensor | None:
|
|
"""Return the first tensor in the payload, or None if empty."""
|
|
if self.tensors:
|
|
return next(iter(self.tensors.values()))
|
|
return None
|
|
|
|
@property
|
|
def is_empty(self) -> bool:
|
|
"""Return True if the payload has no tensors and no metadata."""
|
|
return len(self.tensors) == 0 and len(self.metadata) == 0
|
|
|
|
def get(self, key: str, default: Any = None) -> Any:
|
|
"""Get a value by key, searching tensors first then metadata."""
|
|
if key in self.tensors:
|
|
return self.tensors[key]
|
|
return self.metadata.get(key, default)
|
|
|
|
def __contains__(self, key: str) -> bool:
|
|
return key in self.tensors or key in self.metadata
|
|
|
|
def __getitem__(self, key: str) -> Any:
|
|
"""Dict-like indexing: search tensors first, then metadata."""
|
|
if key in self.tensors:
|
|
return self.tensors[key]
|
|
if key in self.metadata:
|
|
return self.metadata[key]
|
|
raise KeyError(key)
|
|
|
|
def __len__(self) -> int:
|
|
return len(self.tensors) + len(self.metadata)
|
|
|
|
def __iter__(self) -> Iterator[str]:
|
|
yield from self.tensors
|
|
yield from self.metadata
|
|
|
|
def __bool__(self) -> bool:
|
|
return bool(self.tensors) or bool(self.metadata)
|
|
|
|
def __eq__(self, other: object) -> bool:
|
|
"""Support equality with plain dicts and other Mappings."""
|
|
if isinstance(other, MultimodalPayload):
|
|
return self.tensors == other.tensors and self.metadata == other.metadata
|
|
if isinstance(other, Mapping):
|
|
return dict(self) == dict(other)
|
|
return NotImplemented
|
|
|
|
def merged_with(self, incoming: MultimodalPayload) -> MultimodalPayload:
|
|
"""Merge *incoming* onto this payload and return the result.
|
|
|
|
Tensor values accumulate into lists for deferred concatenation;
|
|
non-tensor values are replaced with the latest. When this payload
|
|
is empty, *incoming* is returned as-is, so callers should use the
|
|
return value: ``accumulated = accumulated.merged_with(incoming)``.
|
|
"""
|
|
if self.is_empty:
|
|
return incoming
|
|
_append_entries(self.tensors, incoming.tensors)
|
|
_append_entries(self.metadata, incoming.metadata)
|
|
return self
|
|
|
|
def consolidate_tensors(self, strategy: TensorAccumulationStrategy) -> None:
|
|
"""Concatenate deferred tensor lists into single tensors.
|
|
|
|
Tensors are generated content accumulated as chunks, so lists are
|
|
concatenated according to *strategy* (e.g. audio chunks along the
|
|
time dimension, latent frames along the batch dimension).
|
|
"""
|
|
# Relocate scalar metadata tensors (e.g. sample rate) that from_dict
|
|
# routed into .tensors, so they take the REPLACE path via .metadata
|
|
# instead of failing torch.cat as 0-d tensors.
|
|
for key in _METADATA_TENSOR_KEYS:
|
|
value = self.tensors.pop(key, None)
|
|
if value is None:
|
|
continue
|
|
self.metadata[key] = value[-1] if isinstance(value, list) else value
|
|
|
|
for key, value in list(self.tensors.items()):
|
|
if _is_tensor_list(value):
|
|
self.tensors[key] = _consolidate_tensor_list(key, value, strategy)
|
|
|
|
def consolidate_metadata(self) -> None:
|
|
"""Resolve deferred tensor lists in metadata by keeping the latest value.
|
|
|
|
Metadata values are per-step snapshots (e.g. sample rate), not
|
|
content deltas, so the latest value supersedes earlier ones.
|
|
Nested dicts (unflattened payloads) are resolved one level down.
|
|
"""
|
|
for key, value in list(self.metadata.items()):
|
|
if _is_tensor_list(value):
|
|
self.metadata[key] = value[-1]
|
|
elif isinstance(value, dict):
|
|
for sub_key, sub_value in list(value.items()):
|
|
if _is_tensor_list(sub_value):
|
|
value[sub_key] = sub_value[-1]
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
"""Convert back to a plain dict (tensors + metadata merged)."""
|
|
result: dict[str, Any] = dict(self.tensors)
|
|
result.update(self.metadata)
|
|
return result
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: dict[str, Any] | None) -> MultimodalPayload | None:
|
|
"""Create a MultimodalPayload from a raw dictionary.
|
|
|
|
Separates torch.Tensor values into tensors and everything
|
|
else into metadata.
|
|
"""
|
|
if not data:
|
|
return None
|
|
tensors: dict[str, torch.Tensor] = {}
|
|
metadata: dict[str, Any] = {}
|
|
for k, v in data.items():
|
|
if isinstance(v, torch.Tensor):
|
|
tensors[k] = v
|
|
else:
|
|
metadata[k] = v
|
|
if not tensors and not metadata:
|
|
return None
|
|
return cls(tensors=tensors, metadata=metadata)
|
|
|
|
@classmethod
|
|
def from_raw(cls, payload: Any, modality_key: str) -> MultimodalPayload | None:
|
|
"""Create a MultimodalPayload from a raw producer payload.
|
|
|
|
Accepts a MultimodalPayload (returned as-is), a dict, or a bare
|
|
tensor (stored under *modality_key*). Tensors are moved to CPU.
|
|
Producer-specific dict keys are remapped to the semantic modality
|
|
key (e.g. "audio", "latent"): AR runners produce {"hidden": ...}
|
|
and generation runners produce {"model_outputs": ...}.
|
|
"""
|
|
if isinstance(payload, MultimodalPayload):
|
|
return payload
|
|
|
|
if not isinstance(payload, dict):
|
|
return cls.from_dict({modality_key: _to_cpu(payload)})
|
|
|
|
remapped: dict[str, Any] = {}
|
|
for key, value in payload.items():
|
|
is_producer_key = key == "model_outputs" or (key == "hidden" and modality_key != "hidden")
|
|
remapped[modality_key if is_producer_key else key] = _to_cpu(value)
|
|
return cls.from_dict(remapped)
|
|
|
|
|
|
@dataclass
|
|
class MultimodalCompletionOutput(CompletionOutput):
|
|
"""CompletionOutput with multimodal support.
|
|
|
|
Inherits all CompletionOutput fields and adds multimodal_output.
|
|
As a CompletionOutput subclass, compatible with all existing vLLM consumers.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
multimodal_output: MultimodalPayload | None = None,
|
|
**kwargs: Any,
|
|
):
|
|
super().__init__(**kwargs)
|
|
self.multimodal_output = multimodal_output
|
|
|
|
def __repr__(self) -> str:
|
|
base = super().__repr__()
|
|
return f"{base[:-1]}, multimodal_output={self.multimodal_output!r})"
|