[Fix] JoyVL serving: align with reference engine (bounded long-term memory, timestamps, max_pixels) + recipe updates (#4623)

Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-06-24 07:04:17 +08:00
1 parent 0958e62ecc
commit d35cdd2dab
9 files changed
+116 -28

No files matched your search

@@ -33,6 +33,7 @@ class QAEntry:
@dataclass
class SessionMemory:
long_term_memory: str = ""
long_term_blocks: list[str] = field(default_factory=list)
mid_term_summaries: list[MidTermSummary] = field(default_factory=list)
qa_history: list[QAEntry] = field(default_factory=list)
@@ -91,6 +92,35 @@ def _sample_indices(n: int, budget: int) -> list[int]:
return selected
def _resize_data_url(data_url: str, max_pixels: int) -> str:
"""Downscale a base64 image data URL so width*height <= max_pixels (preserving
aspect ratio), mirroring the reference summarizer. No-op if already within budget,
not a data URL, or PIL is unavailable."""
if max_pixels <= 0 or not data_url.startswith("data:"):
return data_url
import base64
import io
import re
match = re.match(r"data:image/\w+;base64,(.+)", data_url)
if not match:
return data_url
try:
from PIL import Image
img = Image.open(io.BytesIO(base64.b64decode(match.group(1))))
w, h = img.size
if w * h <= max_pixels:
return data_url
scale = (max_pixels / (w * h)) ** 0.5
img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
buf = io.BytesIO()
img.convert("RGB").save(buf, format="JPEG")
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
except Exception:
return data_url
class Summarizer:
def __init__(
self,
@@ -100,12 +130,14 @@ class Summarizer:
mid_term_max_tokens: int = 4000,
long_term_max_tokens: int = 4000,
preferred_time_span: float = 10.0,
max_pixels: int = 262144,
) -> None:
self._backend = backend
self._key_frames = key_frames_per_chunk
self._mid_max_tokens = mid_term_max_tokens
self._long_max_tokens = long_term_max_tokens
self._preferred_time_span = preferred_time_span
self._max_pixels = max_pixels
async def summarize_chunk(
self,
@@ -126,24 +158,25 @@ class Summarizer:
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
for time_range, data_url in picked:
content.append({"type": "text", "text": f"<{time_range}>"})
content.append({"type": "image_url", "image_url": {"url": data_url}})
content.append({"type": "image_url", "image_url": {"url": _resize_data_url(data_url, self._max_pixels)}})
text, _ = await self._backend.generate(
[{"role": "user", "content": content}],
max_tokens=self._mid_max_tokens,
temperature=0.8,
top_p=0.9,
extra_body={"top_k": 40, "repetition_penalty": 1.1, "presence_penalty": 1.0},
extra_body={"top_k": 40, "repetition_penalty": 1.1, "presence_penalty": 1.0, "greedy": False},
)
return text.strip()
async def compress_to_long_term(
self,
existing_long_term: str,
mid_terms: list[MidTermSummary],
) -> str:
async def compress_to_long_term(self, mid_terms: list[MidTermSummary]) -> str:
"""Compress a batch of mid-term summaries into a single long-term block.
Returns one block; the caller (InteractionBrain) accumulates blocks and applies
the sliding window, matching the reference long-term memory behavior.
"""
if not mid_terms:
return existing_long_term
return ""
merged_range = f"{_range_start(mid_terms[0].frame_range)}-{_range_end(mid_terms[-1].frame_range)}"
summaries_text = "\n\n".join(f"<{m.frame_range}>\n{m.summary_text}" for m in mid_terms)
@@ -157,10 +190,9 @@ class Summarizer:
max_tokens=self._long_max_tokens,
temperature=0.3,
top_p=0.7,
extra_body={"top_k": 30, "repetition_penalty": 1.1, "presence_penalty": 0.5},
extra_body={"top_k": 30, "repetition_penalty": 1.1, "presence_penalty": 0.5, "greedy": False},
)
block = f"<{merged_range}>\n{compressed.strip()}"
return f"{existing_long_term.rstrip()}\n\n{block}" if existing_long_term else block
return f"<{merged_range}>\n{compressed.strip()}"
async def aclose(self) -> None:
await self._backend.aclose()