# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Text-only LLM judge for OmniInteract unified evaluation. The three prompt templates follow the official English judge protocol in Lucky-Lance/OmniInteract ``eval/evaluation/llm_judge.py`` (commit de304cef35fd9a50a5caadb5090c34cfbf0dd868). They correspond to the OmniInteract paper appendix (arXiv:2605.26485) Listing A.1 (early-stage), Listing A.2 (interrupted partial quality), and Listing A.3 (core-stage). A local OpenAI-compatible judge is not the paper's GPT-4o judge, so scores are protocol-compatible rather than official paper-table numbers. """ from __future__ import annotations import json import math import threading import time from collections.abc import Mapping from dataclasses import dataclass import regex as re import requests _JUDGE_MAX_ATTEMPTS = 3 _RETRYABLE_HTTP_STATUS = frozenset({502, 503, 504}) _SCORE_FLOAT_PATTERN = re.compile( r"""score["']?\s*[:=]\s*([-+]?\d*\.?\d+(?:[eE][-+]?\d+)?)""", re.IGNORECASE, ) # Official English templates from OmniInteract llm_judge.py. # Paper appendix: Listing A.1 early, A.2 interrupted-partial, A.3 core. # Source: Lucky-Lance/OmniInteract@de304cef35fd9a50a5caadb5090c34cfbf0dd868 EARLY_SYSTEM_PROMPT = ( "You are a streaming voice assistant evaluation judge. " "Judge only based on the given text. " "Output must be parseable JSON with no other text." ) EARLY_USER_TEMPLATE = """Determine whether the early output between start and t_a is an early hallucination. [scene_type] {scene_type} [slot] slot_id={slot_id}, turn_index={turn_index}, step_index={step_index}, boundary_type={boundary_type}, is_interrupted={is_interrupted} [question] {question} [current_gt_answer] {gt_answer} [full_chunk_context] {full_context} [early_actual_text] {actual_text} Rules: 1. Greetings, confirmations, waiting, brief observations, and follow-up phrases -> Neutral. 2. If the model starts substantively answering, guessing unseen info, revealing future steps, or making definitive factual claims -> FP. 3. For 1QnA first step, reciting the full procedure before acting -> FP. 4. score is interaction quality 0-1 when Neutral; 0 when hallucination. Output JSON: {{"flag":"Neutral|FP_Hallucination", "score":float 0-1, "rationale":"one sentence"}}""" CORE_SYSTEM_PROMPT = ( "You are a strict streaming voice assistant core-answer evaluation judge. " "Judge only based on the given text and reference answer. " "Output must be parseable JSON with no other text." ) CORE_USER_TEMPLATE = """Score the core output after t_a. [scene_type] {scene_type} [slot] slot_id={slot_id}, turn_index={turn_index}, step_index={step_index}, boundary_type={boundary_type}, is_interrupted={is_interrupted} [question] {question} [current_gt_answer] {gt_answer} [future_gt_answers_or_steps] {future_answers} [full_chunk_context] {full_context} [core_actual_text] {actual_text} Rules: 1. score 0-1: correctness and coverage of core_actual_text vs gt_answer. 2. Off-topic, factual errors, or missing key answer -> low score. 3. 1QnA: reward only current-step info; penalize spoiling future steps or skipping the current step. 4. If score > 0, extract the earliest contiguous substring from core_actual_text that establishes the answer as trigger_phrase. 5. trigger_phrase must be a verbatim substring; empty if score == 0. Output JSON: {{"score":float 0-1, "trigger_phrase":"substring or empty", "spoiler":true|false, "rationale":"one sentence"}}""" INTERRUPT_PARTIAL_SYSTEM_PROMPT = ( "You are a strict evaluator for interrupted voice-assistant answers. " "Judge only from the provided text. " "Return valid JSON only, with no extra text." ) INTERRUPT_PARTIAL_USER_TEMPLATE = """Evaluate the quality of the assistant output that was already spoken before or around an interruption. [Task] The assistant was answering, but the interaction was interrupted before completion. The assistant was not required to complete the full original answer. Score whether the content already spoken is relevant, correct, and useful for the current ground-truth answer. [Question] {question} [Ground Truth Answer] {gt_answer} [Assistant Output Already Spoken] {actual_text} Scoring Rules: 1. Score from 0 to 1. 2. Do not penalize incompleteness: a partial answer can receive a high score if the spoken part is correct and useful. 3. Score high when the spoken content overlaps with, paraphrases, or conveys useful parts of the ground truth. 4. Score low for acknowledgments or prefaces without substantive answer content. 5. Score low for wrong-question, irrelevant, or generic-filler output. 6. hallucination=true if the output contains clear incorrect facts, wrong target content, or unsupported content. 7. Ignore overflow duration when scoring quality; spill is measured separately. Output JSON: {{"score":float 0-1, "hallucination":true|false, "rationale":"one sentence"}}""" class JudgeRequestError(RuntimeError): """Raised when the configured judge endpoint cannot produce a response.""" @dataclass(frozen=True) class EarlyJudgment: category: str score: float rationale: str raw: str parse_source: str @dataclass(frozen=True) class CoreJudgment: score: float trigger_phrase: str spoiler: bool rationale: str raw: str parse_source: str @dataclass(frozen=True) class PartialJudgment: score: float hallucination: bool rationale: str raw: str parse_source: str def _clamp_unit_interval(value: float) -> float: return max(0.0, min(1.0, value)) def _json_unit_score(value: object) -> tuple[float | None, bool]: """Parse one JSON score into a finite unit-interval value. The second flag is true when the value is a non-finite float (NaN or infinity). Callers must treat that as a parse failure and award no credit. A value that is not a float returns ``(None, False)`` so existing fallbacks stay available. """ if isinstance(value, bool) or not isinstance(value, int | float | str): return None, False try: parsed = float(value) except ValueError: return None, False if not math.isfinite(parsed): return None, True return _clamp_unit_interval(parsed), False def _required_json_unit_score(parsed: Mapping[str, object]) -> tuple[float | None, bool]: """Require an explicit finite ``score`` field on a parsed JSON object. Returns ``(score, False)`` for a present, finite unit-interval value (including an explicit ``0`` / ``0.0``). Returns ``(None, True)`` when ``score`` is missing, has an invalid type, or is non-finite — callers must treat that as ``llm_parse_failed`` and must not invent a valid zero. """ if "score" not in parsed: return None, True score, non_finite = _json_unit_score(parsed["score"]) if non_finite or score is None: return None, True return score, False def _text(value: object) -> str: return str(value or "").strip() def _bool(value: object) -> bool: if isinstance(value, bool): return value if isinstance(value, int | float): return bool(value) return _text(value).lower() in {"1", "true", "yes", "y", "t"} def _first_json_object(text: str) -> dict[str, object] | None: decoder = json.JSONDecoder() position = text.find("{") while position >= 0: try: value, _ = decoder.raw_decode(text[position:]) except json.JSONDecodeError: position = text.find("{", position + 1) continue if isinstance(value, dict) and all(isinstance(key, str) for key in value): return value position = text.find("{", position + 1) return None def _score_float_from_text(text: str) -> float | None: """Parse a unit-interval score from text; ignore unrelated leading numbers.""" match = _SCORE_FLOAT_PATTERN.search(text) if match is None: return None try: value = float(match.group(1)) except ValueError: return None if 0.0 <= value <= 1.0: return value return None def _chat_url(base_url: str) -> str: base = base_url.rstrip("/") if base.endswith("/chat/completions"): return base if base.endswith("/v1"): return f"{base}/chat/completions" return f"{base}/v1/chat/completions" class OmniInteractJudge: """OpenAI-compatible client implementing the OmniInteract judge contract.""" def __init__( self, base_url: str, model: str, *, api_key: str = "EMPTY", timeout_s: float = 60.0, max_tokens: int = 512, ) -> None: if not base_url.strip(): raise ValueError("judge base URL is required") if not model.strip(): raise ValueError("judge model is required") self.base_url = base_url self.model = model self.api_key = api_key or "EMPTY" self.timeout_s = timeout_s self.max_tokens = max_tokens self._thread_local = threading.local() def _session(self) -> requests.Session: session = getattr(self._thread_local, "session", None) if session is None: session = requests.Session() self._thread_local.session = session return session def _generate(self, system_prompt: str, user_prompt: str) -> str: headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", } payload = { "model": self.model, "messages": [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], "temperature": 0.0, "stream": False, "max_tokens": self.max_tokens, } last_error: Exception | None = None for attempt in range(_JUDGE_MAX_ATTEMPTS): try: response = self._session().post( _chat_url(self.base_url), headers=headers, json=payload, timeout=self.timeout_s, ) except requests.Timeout as exc: last_error = JudgeRequestError(f"judge request failed: {exc}") if attempt + 1 < _JUDGE_MAX_ATTEMPTS: time.sleep(0.5 * (2**attempt)) continue raise last_error from exc except requests.RequestException as exc: raise JudgeRequestError(f"judge request failed: {exc}") from exc if response.status_code in _RETRYABLE_HTTP_STATUS and attempt + 1 < _JUDGE_MAX_ATTEMPTS: time.sleep(0.5 * (2**attempt)) continue if not response.ok: raise JudgeRequestError(f"judge returned HTTP {response.status_code}: {response.text[:500]}") try: body = response.json() choices = body["choices"] message = choices[0]["message"] return str(message["content"]) except (KeyError, IndexError, TypeError, ValueError) as exc: raise JudgeRequestError(f"unexpected judge response: {response.text[:500]}") from exc assert last_error is not None raise last_error def judge_early( self, slot: Mapping[str, object], full_context: str, actual_text: str, ) -> EarlyJudgment: raw = self._generate( EARLY_SYSTEM_PROMPT, EARLY_USER_TEMPLATE.format( scene_type=_text(slot.get("scene_type")), slot_id=slot.get("slot_id"), turn_index=slot.get("turn_index"), step_index=slot.get("step_index"), boundary_type=_text(slot.get("boundary_type")), is_interrupted=str(_bool(slot.get("is_interrupted"))).lower(), question=_text(slot.get("question_text")), gt_answer=_text(slot.get("gt_answer")), full_context=full_context.strip(), actual_text=actual_text.strip(), ), ) parsed = _first_json_object(raw) flag = _text(parsed.get("flag")) if parsed else raw.strip() normalized = flag.lower().replace("-", "_").replace(" ", "") if "hallucination" in normalized: return EarlyJudgment( category="hallucination", score=0.0, rationale=_text(parsed.get("rationale")) if parsed else "", raw=raw, parse_source="llm_json" if parsed else "llm_parse_failed", ) if "neutral" in normalized: if parsed is None: return EarlyJudgment( category="neutral", score=0.0, rationale="", raw=raw, parse_source="llm_parse_failed", ) score, score_failed = _required_json_unit_score(parsed) if score_failed: return EarlyJudgment( category="unparsed", score=0.0, rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_parse_failed", ) return EarlyJudgment( category="neutral", score=score if score is not None else 0.0, rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_json", ) return EarlyJudgment( category="unparsed", score=0.0, rationale=_text(parsed.get("rationale")) if parsed else "", raw=raw, parse_source="llm_parse_failed", ) def judge_core( self, slot: Mapping[str, object], full_context: str, actual_text: str, future_answers: str, ) -> CoreJudgment: raw = self._generate( CORE_SYSTEM_PROMPT, CORE_USER_TEMPLATE.format( scene_type=_text(slot.get("scene_type")), slot_id=slot.get("slot_id"), turn_index=slot.get("turn_index"), step_index=slot.get("step_index"), boundary_type=_text(slot.get("boundary_type")), is_interrupted=str(_bool(slot.get("is_interrupted"))).lower(), question=_text(slot.get("question_text")), gt_answer=_text(slot.get("gt_answer")), future_answers=future_answers.strip(), full_context=full_context.strip(), actual_text=actual_text.strip(), ), ) parsed = _first_json_object(raw) if parsed is not None: score, json_score_failed = _required_json_unit_score(parsed) if json_score_failed: return CoreJudgment( score=0.0, trigger_phrase="", spoiler=False, rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_parse_failed", ) return CoreJudgment( score=score if score is not None else 0.0, trigger_phrase=_text(parsed.get("trigger_phrase")), spoiler=_bool(parsed.get("spoiler")), rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_json", ) fallback = _score_float_from_text(raw) return CoreJudgment( score=fallback if fallback is not None else 0.0, trigger_phrase="", spoiler=False, rationale="", raw=raw, parse_source="llm_float_text" if fallback is not None else "llm_parse_failed", ) def judge_interrupted_partial( self, slot: Mapping[str, object], actual_text: str, ) -> PartialJudgment: raw = self._generate( INTERRUPT_PARTIAL_SYSTEM_PROMPT, INTERRUPT_PARTIAL_USER_TEMPLATE.format( question=_text(slot.get("question_text")), gt_answer=_text(slot.get("gt_answer")), actual_text=actual_text.strip(), ), ) parsed = _first_json_object(raw) if parsed is None: return PartialJudgment( score=0.0, hallucination=False, rationale="", raw=raw, parse_source="llm_parse_failed", ) score, score_failed = _required_json_unit_score(parsed) if score_failed: return PartialJudgment( score=0.0, hallucination=False, rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_parse_failed", ) return PartialJudgment( score=score if score is not None else 0.0, hallucination=_bool(parsed.get("hallucination")), rationale=_text(parsed.get("rationale")), raw=raw, parse_source="llm_json", )