[Fix] JoyVL serving: align with reference engine (bounded long-term memory, timestamps, max_pixels) + recipe updates (#4623)
Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
0958e62ecc
commit
d35cdd2dab
9 files changed
+116
-28
No files matched your search
@@ -33,6 +33,7 @@ class QAEntry:
|
||||
@dataclass
|
||||
class SessionMemory:
|
||||
long_term_memory: str = ""
|
||||
long_term_blocks: list[str] = field(default_factory=list)
|
||||
mid_term_summaries: list[MidTermSummary] = field(default_factory=list)
|
||||
qa_history: list[QAEntry] = field(default_factory=list)
|
||||
|
||||
@@ -91,6 +92,35 @@ def _sample_indices(n: int, budget: int) -> list[int]:
|
||||
return selected
|
||||
|
||||
|
||||
def _resize_data_url(data_url: str, max_pixels: int) -> str:
|
||||
"""Downscale a base64 image data URL so width*height <= max_pixels (preserving
|
||||
aspect ratio), mirroring the reference summarizer. No-op if already within budget,
|
||||
not a data URL, or PIL is unavailable."""
|
||||
if max_pixels <= 0 or not data_url.startswith("data:"):
|
||||
return data_url
|
||||
import base64
|
||||
import io
|
||||
import re
|
||||
|
||||
match = re.match(r"data:image/\w+;base64,(.+)", data_url)
|
||||
if not match:
|
||||
return data_url
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(io.BytesIO(base64.b64decode(match.group(1))))
|
||||
w, h = img.size
|
||||
if w * h <= max_pixels:
|
||||
return data_url
|
||||
scale = (max_pixels / (w * h)) ** 0.5
|
||||
img = img.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
img.convert("RGB").save(buf, format="JPEG")
|
||||
return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
|
||||
except Exception:
|
||||
return data_url
|
||||
|
||||
|
||||
class Summarizer:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -100,12 +130,14 @@ class Summarizer:
|
||||
mid_term_max_tokens: int = 4000,
|
||||
long_term_max_tokens: int = 4000,
|
||||
preferred_time_span: float = 10.0,
|
||||
max_pixels: int = 262144,
|
||||
) -> None:
|
||||
self._backend = backend
|
||||
self._key_frames = key_frames_per_chunk
|
||||
self._mid_max_tokens = mid_term_max_tokens
|
||||
self._long_max_tokens = long_term_max_tokens
|
||||
self._preferred_time_span = preferred_time_span
|
||||
self._max_pixels = max_pixels
|
||||
|
||||
async def summarize_chunk(
|
||||
self,
|
||||
@@ -126,24 +158,25 @@ class Summarizer:
|
||||
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
||||
for time_range, data_url in picked:
|
||||
content.append({"type": "text", "text": f"<{time_range}>"})
|
||||
content.append({"type": "image_url", "image_url": {"url": data_url}})
|
||||
content.append({"type": "image_url", "image_url": {"url": _resize_data_url(data_url, self._max_pixels)}})
|
||||
|
||||
text, _ = await self._backend.generate(
|
||||
[{"role": "user", "content": content}],
|
||||
max_tokens=self._mid_max_tokens,
|
||||
temperature=0.8,
|
||||
top_p=0.9,
|
||||
extra_body={"top_k": 40, "repetition_penalty": 1.1, "presence_penalty": 1.0},
|
||||
extra_body={"top_k": 40, "repetition_penalty": 1.1, "presence_penalty": 1.0, "greedy": False},
|
||||
)
|
||||
return text.strip()
|
||||
|
||||
async def compress_to_long_term(
|
||||
self,
|
||||
existing_long_term: str,
|
||||
mid_terms: list[MidTermSummary],
|
||||
) -> str:
|
||||
async def compress_to_long_term(self, mid_terms: list[MidTermSummary]) -> str:
|
||||
"""Compress a batch of mid-term summaries into a single long-term block.
|
||||
|
||||
Returns one block; the caller (InteractionBrain) accumulates blocks and applies
|
||||
the sliding window, matching the reference long-term memory behavior.
|
||||
"""
|
||||
if not mid_terms:
|
||||
return existing_long_term
|
||||
return ""
|
||||
|
||||
merged_range = f"{_range_start(mid_terms[0].frame_range)}-{_range_end(mid_terms[-1].frame_range)}"
|
||||
summaries_text = "\n\n".join(f"<{m.frame_range}>\n{m.summary_text}" for m in mid_terms)
|
||||
@@ -157,10 +190,9 @@ class Summarizer:
|
||||
max_tokens=self._long_max_tokens,
|
||||
temperature=0.3,
|
||||
top_p=0.7,
|
||||
extra_body={"top_k": 30, "repetition_penalty": 1.1, "presence_penalty": 0.5},
|
||||
extra_body={"top_k": 30, "repetition_penalty": 1.1, "presence_penalty": 0.5, "greedy": False},
|
||||
)
|
||||
block = f"<{merged_range}>\n{compressed.strip()}"
|
||||
return f"{existing_long_term.rstrip()}\n\n{block}" if existing_long_term else block
|
||||
return f"<{merged_range}>\n{compressed.strip()}"
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._backend.aclose()
|
||||
|
||||
Reference in new issue
Block a user