Update vLLM wheel version in Dockerfile and enhance input processing for omni models. Added functionality to reinject omni-specific fields into processed prompts and improved handling of multimodal audio extraction in serving_chat.py. Adjusted SharedMemoryConnector logic for payload size checks and refined input preprocessing methods for better compatibility with multimodal data.

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-02-20 08:34:53 +00:00
1 parent 663976b8ca
commit 9869ea007e
8 files changed
+214 -96

No files matched your search

@@ -35,12 +35,26 @@ def try_send_via_connector(
try:
t0 = time.time()
# Strip non-serializable multimodal feature fields from original_prompt
# before including it in metadata. After stage-0 runs, the TokPrompt
# returned by render_chat_async may carry processed multimodal features
# (mm_kwargs, mm_placeholders, mm_hashes) that contain MultiModalKwargsItems
# objects, which are not supported by OmniMsgpackEncoder. The receiving
# side (try_recv_via_connector) only extracts "engine_inputs" from the
# payload and never uses "original_prompt", so stripping these fields
# only affects debug metadata and is safe.
_MM_FEATURE_KEYS = frozenset({"mm_kwargs", "mm_placeholders", "mm_hashes"})
if isinstance(original_prompt, dict) and any(k in original_prompt for k in _MM_FEATURE_KEYS):
safe_prompt = {k: v for k, v in original_prompt.items() if k not in _MM_FEATURE_KEYS}
else:
safe_prompt = original_prompt
# Prepare data for connector
payload_data = {
"engine_inputs": next_inputs,
"sampling_params": sampling_params,
"metadata": {
"original_prompt": original_prompt,
"original_prompt": safe_prompt,
"stage_transition": f"{stage_id}->{next_stage_id}",
"timestamp": time.time(),
},
@@ -48,7 +48,7 @@ class SharedMemoryConnector(OmniConnectorBase):
payload = self.serialize_obj(data)
size = len(payload)
if True:
if size > self.threshold:
# Use Shared Memory
lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock"
with open(lock_file, "w") as lockf: