Update vLLM wheel version in Dockerfile and enhance input processing for omni models. Added functionality to reinject omni-specific fields into processed prompts and improved handling of multimodal audio extraction in serving_chat.py. Adjusted SharedMemoryConnector logic for payload size checks and refined input preprocessing methods for better compatibility with multimodal data.
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
663976b8ca
commit
9869ea007e
8 files changed
+214
-96
No files matched your search
@@ -35,12 +35,26 @@ def try_send_via_connector(
|
||||
try:
|
||||
t0 = time.time()
|
||||
|
||||
# Strip non-serializable multimodal feature fields from original_prompt
|
||||
# before including it in metadata. After stage-0 runs, the TokPrompt
|
||||
# returned by render_chat_async may carry processed multimodal features
|
||||
# (mm_kwargs, mm_placeholders, mm_hashes) that contain MultiModalKwargsItems
|
||||
# objects, which are not supported by OmniMsgpackEncoder. The receiving
|
||||
# side (try_recv_via_connector) only extracts "engine_inputs" from the
|
||||
# payload and never uses "original_prompt", so stripping these fields
|
||||
# only affects debug metadata and is safe.
|
||||
_MM_FEATURE_KEYS = frozenset({"mm_kwargs", "mm_placeholders", "mm_hashes"})
|
||||
if isinstance(original_prompt, dict) and any(k in original_prompt for k in _MM_FEATURE_KEYS):
|
||||
safe_prompt = {k: v for k, v in original_prompt.items() if k not in _MM_FEATURE_KEYS}
|
||||
else:
|
||||
safe_prompt = original_prompt
|
||||
|
||||
# Prepare data for connector
|
||||
payload_data = {
|
||||
"engine_inputs": next_inputs,
|
||||
"sampling_params": sampling_params,
|
||||
"metadata": {
|
||||
"original_prompt": original_prompt,
|
||||
"original_prompt": safe_prompt,
|
||||
"stage_transition": f"{stage_id}->{next_stage_id}",
|
||||
"timestamp": time.time(),
|
||||
},
|
||||
|
||||
@@ -48,7 +48,7 @@ class SharedMemoryConnector(OmniConnectorBase):
|
||||
payload = self.serialize_obj(data)
|
||||
size = len(payload)
|
||||
|
||||
if True:
|
||||
if size > self.threshold:
|
||||
# Use Shared Memory
|
||||
lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock"
|
||||
with open(lock_file, "w") as lockf:
|
||||
|
||||
Reference in new issue
Block a user