# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

from __future__ import annotations

import json
from dataclasses import asdict
from pathlib import Path

import pytest

from vllm_omni.benchmarks.data_modules.omniinteract_dataset import (
    OmniInteractCase,
    OmniInteractEvaluationOptions,
)
from vllm_omni.benchmarks.omniinteract import OmniInteractCaseResult
from vllm_omni.benchmarks.omniinteract_eval import (
    AlignedWord,
    TranscriptChunk,
    _summarize,
    build_slots,
    evaluate_batch,
    evaluate_case,
    evaluation_inputs_fingerprint,
    match_slots,
)
from vllm_omni.benchmarks.omniinteract_judge import (
    CoreJudgment,
    EarlyJudgment,
    OmniInteractJudge,
    PartialJudgment,
)

pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]


class _FixedJudge:
    def judge_early(
        self,
        slot: dict[str, object],
        full_context: str,
        actual_text: str,
    ) -> EarlyJudgment:
        del slot, full_context, actual_text
        return EarlyJudgment("neutral", 1.0, "ok", "{}", "llm_json")

    def judge_core(
        self,
        slot: dict[str, object],
        full_context: str,
        actual_text: str,
        future_answers: str,
    ) -> CoreJudgment:
        del slot, full_context, actual_text, future_answers
        return CoreJudgment(1.0, "answer", False, "ok", "{}", "llm_json")

    def judge_interrupted_partial(
        self,
        slot: dict[str, object],
        actual_text: str,
    ) -> PartialJudgment:
        del slot, actual_text
        return PartialJudgment(1.0, False, "ok", "{}", "llm_json")


class _GeneratedJudge(OmniInteractJudge):
    def __init__(self, responses: list[str]) -> None:
        super().__init__("http://judge", "judge-model")
        self.responses = iter(responses)

    def _generate(self, system_prompt: str, user_prompt: str) -> str:
        del system_prompt, user_prompt
        return next(self.responses)


def test_build_slots_and_split_word_aligned_chunk() -> None:
    slots = build_slots(
        [
            {
                "question_time": 0.0,
                "answer_time": 2.0,
                "question_text": "question",
                "answer_text": "answer",
                "question_type": "realtime",
            },
            {
                "question_time": 5.0,
                "answer_time": 6.0,
                "question_text": "next",
                "answer_text": "next answer",
                "question_type": "proactive",
            },
        ],
        "multi_turn",
    )
    chunk = TranscriptChunk(
        source_id=0,
        text="okay answer",
        start=0.5,
        end=2.5,
        aligned_words=(
            AlignedWord("okay", 0.5, 0.9),
            AlignedWord("answer", 2.1, 2.5),
        ),
    )

    matched, unmatched = match_slots(slots, [chunk])

    assert not unmatched
    assert matched[0].early_chunks[0].text == "okay"
    assert matched[0].core_chunks[0].text == "answer"
    assert matched[0].core_chunks[0].start == pytest.approx(2.1)


def test_judge_parses_json_wrapped_in_model_text() -> None:
    judge = _GeneratedJudge(
        [
            'result: {"flag":"FP_Hallucination","score":1,"rationale":"early"}',
            '```json\n{"score":0.8,"trigger_phrase":"answer","spoiler":false,"rationale":"good"}\n```',
            '{"score":0.7,"hallucination":false,"rationale":"partial"}',
        ]
    )
    slot = {"question_text": "q", "gt_answer": "answer"}

    early = judge.judge_early(slot, "context", "guess")
    core = judge.judge_core(slot, "context", "answer", "(none)")
    partial = judge.judge_interrupted_partial(slot, "answer")

    assert early.category == "hallucination"
    assert early.score == 0.0
    assert core.score == pytest.approx(0.8)
    assert core.trigger_phrase == "answer"
    assert partial.score == pytest.approx(0.7)


def test_evaluate_case_persists_accuracy_artifact(tmp_path: Path) -> None:
    annotation = tmp_path / "annotation.json"
    annotation.write_text(
        json.dumps(
            [
                {
                    "question_time": 0.0,
                    "answer_time": 1.0,
                    "question_text": "question",
                    "answer_text": "answer",
                    "question_type": "realtime",
                }
            ]
        )
    )
    output_dir = tmp_path / "case-output"
    output_dir.mkdir()
    (output_dir / "wav_transcript.json").write_text(
        json.dumps(
            {
                "chunks": [
                    {
                        "timestamp": [1.0, 1.5],
                        "text": "answer",
                        "aligned_words": [{"text": "answer", "start": 1.0, "end": 1.5}],
                    }
                ]
            }
        )
    )
    video = tmp_path / "video.mp4"
    video.touch()
    case = OmniInteractCase("1q1a", "video.mp4", video, annotation, "multi_turn")
    result = OmniInteractCaseResult(
        subset="1q1a",
        video=str(video),
        output_dir=str(output_dir),
        success=True,
        eligible_for_official_eval=True,
    )
    artifact = tmp_path / "evaluation.json"

    evaluation = evaluate_case(case, result, _FixedJudge(), artifact)

    assert evaluation["timing_precision"] == "word-aligned"
    assert evaluation["summary"]["IA_QTF1"] == pytest.approx(1.0)
    paper = evaluation["summary"]["paper_metrics"]["exp_f1"]
    assert paper["all_global"]["IA_QTF1"] == pytest.approx(1.0)
    assert paper["realtime"]["IA_QTF1"] == pytest.approx(1.0)
    assert json.loads(artifact.read_text())["status"] == "ok"
    fingerprint = evaluation["inputs_fingerprint"]
    assert fingerprint["annotation_sha256"]
    assert fingerprint["transcript_sha256"]
    assert fingerprint["protocol_source"]


def _realtime_eval_case(tmp_path: Path, *, transcript_text: str) -> tuple[OmniInteractCase, OmniInteractCaseResult]:
    annotation = tmp_path / "annotation.json"
    annotation.write_text(
        json.dumps(
            [
                {
                    "question_time": 0.0,
                    "answer_time": 1.0,
                    "question_text": "question",
                    "answer_text": "answer",
                    "question_type": "realtime",
                }
            ]
        )
    )
    output_dir = tmp_path / "case-output"
    output_dir.mkdir(exist_ok=True)
    chunks: list[dict[str, object]] = []
    if transcript_text:
        chunks.append({"timestamp": [1.0, 1.5], "text": transcript_text})
    (output_dir / "wav_transcript.json").write_text(json.dumps({"chunks": chunks}))
    video = tmp_path / "video.mp4"
    video.touch()
    case = OmniInteractCase("1q1a", "video.mp4", video, annotation, "multi_turn")
    result = OmniInteractCaseResult(
        subset="1q1a",
        video=str(video),
        output_dir=str(output_dir),
        success=True,
        eligible_for_official_eval=True,
    )
    return case, result


def _eval_options(tmp_path: Path, *, skip_existing: bool) -> OmniInteractEvaluationOptions:
    return OmniInteractEvaluationOptions(
        judge_base_url="http://127.0.0.1:9",
        judge_model="cached-judge",
        judge_api_key="EMPTY",
        judge_timeout_s=60.0,
        judge_max_tokens=32,
        workers=1,
        output_dir=tmp_path / "evaluation",
        skip_existing=skip_existing,
    )


class _ConfiguredFixedJudge(_FixedJudge):
    def __init__(
        self,
        base_url: str,
        model: str,
        *,
        api_key: str = "EMPTY",
        timeout_s: float = 60.0,
        max_tokens: int = 512,
    ) -> None:
        del api_key, timeout_s
        self.base_url = base_url
        self.model = model
        self.max_tokens = max_tokens


def test_skip_existing_reuses_matching_fingerprint(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    import vllm_omni.benchmarks.omniinteract_eval as eval_mod

    case, result = _realtime_eval_case(tmp_path, transcript_text="answer")
    options = _eval_options(tmp_path, skip_existing=False)
    monkeypatch.setattr(eval_mod, "OmniInteractJudge", _ConfiguredFixedJudge)
    first = evaluate_batch([case], [result], options)
    assert first["summary"]["IA_QTF1"] == pytest.approx(1.0)

    calls: list[str] = []

    def _forbidden(
        case: OmniInteractCase,
        result: OmniInteractCaseResult,
        judge: object,
        output_path: Path,
        config: object = None,
        inputs_fingerprint: object = None,
    ) -> dict[str, object]:
        del case, result, judge, output_path, config, inputs_fingerprint
        calls.append("evaluate_case")
        raise AssertionError("matching fingerprint must reuse the cached evaluation")

    monkeypatch.setattr(eval_mod, "evaluate_case", _forbidden)
    reused = evaluate_batch([case], [result], _eval_options(tmp_path, skip_existing=True))
    assert calls == []
    assert reused["summary"]["IA_QTF1"] == pytest.approx(1.0)


def test_skip_existing_rejects_stale_transcript(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    import vllm_omni.benchmarks.omniinteract_eval as eval_mod

    case, result = _realtime_eval_case(tmp_path, transcript_text="answer")
    options = _eval_options(tmp_path, skip_existing=True)
    fingerprint = evaluation_inputs_fingerprint(
        annotation_path=case.annotation_path,
        transcript_path=Path(result.output_dir) / "wav_transcript.json",
        judge_model=options.judge_model,
        judge_base_url=options.judge_base_url,
        judge_max_tokens=options.judge_max_tokens,
    )
    sample_id = f"{case.subset}__{Path(result.output_dir).name}"
    cache_path = options.output_dir / f"{sample_id}.unified_eval.json"
    cache_path.parent.mkdir(parents=True)
    cache_path.write_text(
        json.dumps(
            {
                "status": "ok",
                "sample_id": sample_id,
                "inputs_fingerprint": asdict(fingerprint),
                "summary": {
                    "IA_QTF1": 1.0,
                    "num_slots": 1,
                    "num_unmatched_chunks": 0,
                    "Global_TP": 1,
                    "Global_FP": 0,
                    "Global_FN": 0,
                },
                "slots": [],
            }
        )
    )
    (Path(result.output_dir) / "wav_transcript.json").write_text(json.dumps({"chunks": []}))
    monkeypatch.setattr(eval_mod, "OmniInteractJudge", _ConfiguredFixedJudge)
    fresh = evaluate_batch([case], [result], options)
    assert fresh["summary"]["IA_QTF1"] == pytest.approx(0.0)


class _MissingCoreScoreJudge(_ConfiguredFixedJudge):
    def judge_core(
        self,
        slot: dict[str, object],
        full_context: str,
        actual_text: str,
        future_answers: str,
    ) -> CoreJudgment:
        del slot, full_context, actual_text, future_answers
        return CoreJudgment(0.0, "", False, "Could not evaluate", "{}", "llm_parse_failed")


def test_evaluate_batch_reports_missing_score_as_parse_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    import vllm_omni.benchmarks.omniinteract_eval as eval_mod

    case, result = _realtime_eval_case(tmp_path, transcript_text="answer")
    options = _eval_options(tmp_path, skip_existing=False)
    monkeypatch.setattr(eval_mod, "OmniInteractJudge", _MissingCoreScoreJudge)
    evaluation = evaluate_batch([case], [result], options)
    assert evaluation["status"] == "failed"
    assert evaluation["evaluated"] == 0
    assert evaluation["failed"] == 1
    sample_id = f"{case.subset}__{Path(result.output_dir).name}"
    cached = json.loads((options.output_dir / f"{sample_id}.unified_eval.json").read_text())
    assert cached["status"] == "parse_failed"


class _CoreRationaleNumberJudge(OmniInteractJudge):
    def __init__(
        self,
        base_url: str,
        model: str,
        *,
        api_key: str = "EMPTY",
        timeout_s: float = 60.0,
        max_tokens: int = 512,
    ) -> None:
        super().__init__(base_url, model, api_key=api_key, timeout_s=timeout_s, max_tokens=max_tokens)

    def _generate(self, system_prompt: str, user_prompt: str) -> str:
        del system_prompt, user_prompt
        return '{"rationale":"Cannot score: 1 required reference is missing."}'


def test_evaluate_batch_rejects_core_rationale_number_as_parse_failure(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    import vllm_omni.benchmarks.omniinteract_eval as eval_mod

    case, result = _realtime_eval_case(tmp_path, transcript_text="answer")
    options = _eval_options(tmp_path, skip_existing=False)
    monkeypatch.setattr(eval_mod, "OmniInteractJudge", _CoreRationaleNumberJudge)
    evaluation = evaluate_batch([case], [result], options)
    assert evaluation["status"] == "failed"
    assert evaluation["evaluated"] == 0
    assert evaluation["failed"] == 1
    sample_id = f"{case.subset}__{Path(result.output_dir).name}"
    cached = json.loads((options.output_dir / f"{sample_id}.unified_eval.json").read_text())
    assert cached["status"] == "parse_failed"
    assert cached["slots"][0]["stage_core"]["parse_source"] == "llm_parse_failed"
    assert cached["slots"][0]["stage_core"]["S_core"] == 0.0


def test_skip_existing_rejects_parse_failed_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    import vllm_omni.benchmarks.omniinteract_eval as eval_mod

    case, result = _realtime_eval_case(tmp_path, transcript_text="answer")
    options = _eval_options(tmp_path, skip_existing=True)
    fingerprint = evaluation_inputs_fingerprint(
        annotation_path=case.annotation_path,
        transcript_path=Path(result.output_dir) / "wav_transcript.json",
        judge_model=options.judge_model,
        judge_base_url=options.judge_base_url,
        judge_max_tokens=options.judge_max_tokens,
    )
    sample_id = f"{case.subset}__{Path(result.output_dir).name}"
    cache_path = options.output_dir / f"{sample_id}.unified_eval.json"
    cache_path.parent.mkdir(parents=True)
    cache_path.write_text(
        json.dumps(
            {
                "status": "parse_failed",
                "sample_id": sample_id,
                "inputs_fingerprint": asdict(fingerprint),
                "summary": {
                    "IA_QTF1": 0.0,
                    "num_slots": 1,
                    "num_unmatched_chunks": 0,
                    "Global_TP": 0,
                    "Global_FP": 0,
                    "Global_FN": 1,
                },
                "slots": [
                    {
                        "stage_core": {"parse_source": "llm_parse_failed", "S_core": 0.0},
                        "TP_n": 0.0,
                        "FP_delta": 0,
                        "FN_delta": 1,
                    }
                ],
            }
        )
    )
    monkeypatch.setattr(eval_mod, "OmniInteractJudge", _ConfiguredFixedJudge)
    fresh = evaluate_batch([case], [result], options)
    assert fresh["evaluated"] == 1
    assert fresh["failed"] == 0
    assert fresh["summary"]["IA_QTF1"] == pytest.approx(1.0)
    assert json.loads(cache_path.read_text())["status"] == "ok"


def _slot_row(
    *,
    scene_type: str,
    question_type: str,
    tp_n: float,
    fp: int = 0,
    fn: int = 0,
    nested_role: str | None = None,
    nested_group_id: int | None = None,
    sample_id: str = "s",
    tp_core: float | None = None,
    answer_start: float | None = 1.0,
) -> dict[str, object]:
    core_tp = tp_n if tp_core is None else tp_core
    return {
        "sample_id": sample_id,
        "scene_type": scene_type,
        "question_type": question_type,
        "nested_role": nested_role,
        "nested_group_id": nested_group_id,
        "TP_n": tp_n,
        "TP_core": core_tp,
        "FP_delta": fp,
        "FN_delta": fn,
        "is_interrupted": False,
        "stage_core": {"answer_start": answer_start},
    }


def test_paper_metrics_exclude_nested_from_realtime_and_proactive() -> None:
    summary = _summarize(
        [
            _slot_row(scene_type="multi_turn", question_type="realtime", tp_n=1.0),
            _slot_row(
                scene_type="nested",
                question_type="realtime",
                tp_n=1.0,
                nested_role="inner",
                nested_group_id=1,
                answer_start=0.5,
            ),
            _slot_row(
                scene_type="nested",
                question_type="proactive",
                tp_n=1.0,
                nested_role="outer",
                nested_group_id=1,
                answer_start=1.5,
            ),
            _slot_row(scene_type="multi_turn", question_type="proactive", tp_n=1.0),
            _slot_row(scene_type="1QnA", question_type="step", tp_n=1.0, fn=0),
        ],
        unmatched=0,
    )
    paper = summary["paper_metrics"]["exp_f1"]

    assert paper["realtime"]["num_slots"] == 1
    assert paper["proactive"]["num_slots"] == 1
    assert paper["nested"]["num_slots"] == 2
    assert paper["one_q1a_global"]["num_slots"] == 4
    assert paper["one_qna"]["num_slots"] == 1
    assert paper["all_global"]["num_slots"] == 5
    # All Global includes 1QnA; 1Q1A Global does not.
    assert paper["all_global"]["Global_TP"] == pytest.approx(5.0)
    assert paper["one_q1a_global"]["Global_TP"] == pytest.approx(4.0)
    assert summary["nested"]["NCCS"] == pytest.approx(1.0)
    assert summary["nested"]["missing_q1_count"] == 0
    assert summary["paper_metrics"]["exp_nested"]["missed_outer"] == 0
    assert summary["paper_metrics"]["exp_nested"]["inner_IA_QTF1"] == pytest.approx(1.0)
    assert summary["paper_metrics"]["exp_nested"]["outer_IA_QTF1"] == pytest.approx(1.0)
    assert summary["scenario_case_counts"]["realtime"] == 1
    assert summary["scenario_case_counts"]["proactive"] == 1
    assert summary["scenario_case_counts"]["nested"] == 1
    assert summary["scenario_case_counts"]["1qna"] == 1


def test_nccs_allows_missing_answer_start() -> None:
    summary = _summarize(
        [
            _slot_row(
                scene_type="nested",
                question_type="realtime",
                tp_n=0.8,
                tp_core=0.8,
                nested_role="inner",
                nested_group_id=1,
                answer_start=None,
            ),
            _slot_row(
                scene_type="nested",
                question_type="proactive",
                tp_n=0.8,
                tp_core=0.8,
                nested_role="outer",
                nested_group_id=1,
                answer_start=None,
            ),
        ],
        unmatched=0,
    )
    assert summary["nested"]["success_pairs"] == 1
    assert summary["nested"]["NCCS"] == pytest.approx(0.8)


def test_missed_outer_counts_zero_outer_core() -> None:
    summary = _summarize(
        [
            _slot_row(
                scene_type="nested",
                question_type="realtime",
                tp_n=1.0,
                tp_core=1.0,
                nested_role="inner",
                nested_group_id=1,
                answer_start=0.5,
            ),
            _slot_row(
                scene_type="nested",
                question_type="proactive",
                tp_n=0.0,
                tp_core=0.0,
                fn=1,
                nested_role="outer",
                nested_group_id=1,
                answer_start=1.5,
            ),
        ],
        unmatched=0,
    )
    nested = summary["paper_metrics"]["exp_nested"]
    assert nested["missed_outer"] == 1
    assert nested["NCCS"] == pytest.approx(0.0)
    assert nested["inner_IA_QTF1"] == pytest.approx(1.0)
    assert nested["outer_IA_QTF1"] == pytest.approx(0.0)


def test_cross_boundary_core_sets_effective_start_hint() -> None:
    slots = build_slots(
        [
            {
                "question_time": 0.0,
                "answer_time": 2.0,
                "question_text": "question",
                "answer_text": "answer",
                "question_type": "realtime",
            }
        ],
        "multi_turn",
    )
    chunk = TranscriptChunk(
        source_id=0,
        text="okay answer",
        start=0.5,
        end=2.5,
        aligned_words=(
            AlignedWord("okay", 0.5, 0.9),
            AlignedWord("answer", 2.1, 2.5),
        ),
    )
    matched, unmatched = match_slots(slots, [chunk])
    assert not unmatched
    assert matched[0].core_chunks[0].effective_start_hint == pytest.approx(2.0)


class _NonNeutralEarlyJudge(_FixedJudge):
    def judge_early(
        self,
        slot: dict[str, object],
        full_context: str,
        actual_text: str,
    ) -> EarlyJudgment:
        del slot, full_context, actual_text
        return EarlyJudgment("other", 1.0, "ignored", "{}", "llm_json")


def test_early_tp_ack_only_for_neutral(tmp_path: Path) -> None:
    annotation = tmp_path / "annotation.json"
    annotation.write_text(
        json.dumps(
            [
                {
                    "question_time": 0.0,
                    "answer_time": 2.0,
                    "question_text": "question",
                    "answer_text": "answer",
                    "question_type": "realtime",
                }
            ]
        )
    )
    output_dir = tmp_path / "case-output"
    output_dir.mkdir()
    (output_dir / "wav_transcript.json").write_text(
        json.dumps(
            {
                "chunks": [
                    {
                        "timestamp": [0.5, 0.9],
                        "text": "okay",
                        "aligned_words": [{"text": "okay", "start": 0.5, "end": 0.9}],
                    },
                    {
                        "timestamp": [2.1, 2.5],
                        "text": "answer",
                        "aligned_words": [{"text": "answer", "start": 2.1, "end": 2.5}],
                    },
                ]
            }
        )
    )
    video = tmp_path / "video.mp4"
    video.touch()
    case = OmniInteractCase("1q1a", "video.mp4", video, annotation, "multi_turn")
    result = OmniInteractCaseResult(
        subset="1q1a",
        video=str(video),
        output_dir=str(output_dir),
        success=True,
        eligible_for_official_eval=True,
    )
    evaluation = evaluate_case(case, result, _NonNeutralEarlyJudge(), tmp_path / "evaluation.json")
    assert evaluation["slots"][0]["TP_ack"] == 0.0
    assert evaluation["slots"][0]["stage_early"]["category"] == "other"


def test_paper_metrics_unknown_inner_does_not_underflow_realtime() -> None:
    summary = _summarize(
        [
            _slot_row(scene_type="multi_turn", question_type="realtime", tp_n=0.5),
            _slot_row(
                scene_type="nested",
                question_type="unknown",
                tp_n=2.0,
                nested_role="inner",
                nested_group_id=1,
                answer_start=0.5,
            ),
            _slot_row(
                scene_type="nested",
                question_type="proactive",
                tp_n=1.0,
                nested_role="outer",
                nested_group_id=1,
                answer_start=1.5,
            ),
        ],
        unmatched=0,
    )
    paper = summary["paper_metrics"]["exp_f1"]
    assert paper["realtime"]["Global_TP"] == pytest.approx(0.5)
    assert paper["realtime"]["num_slots"] == 1
    assert paper["realtime"]["Precision"] >= 0.0
    assert paper["one_q1a_global"]["Global_TP"] == pytest.approx(3.5)


def test_core_score_float_ignores_leading_counts() -> None:
    from vllm_omni.benchmarks.omniinteract_judge import _score_float_from_text

    assert _score_float_from_text("The answer covers 3 of the 5 key points, score: 0.6") == pytest.approx(0.6)
    assert _score_float_from_text("score: 3") is None
    assert _score_float_from_text("no score here 0.9") is None

    judge = _GeneratedJudge(["The answer covers 3 of the 5 key points, score: 0.6"])
    core = judge.judge_core({"question_text": "q", "gt_answer": "a"}, "ctx", "answer", "(none)")
    assert core.score == pytest.approx(0.6)
    assert core.parse_source == "llm_float_text"


def test_core_missing_json_score_does_not_use_rationale_number() -> None:
    raw = '{"rationale":"Cannot score: 1 required reference is missing."}'
    core = _GeneratedJudge([raw]).judge_core({"question_text": "q", "gt_answer": "a"}, "ctx", "answer", "(none)")
    assert core.score == 0.0
    assert core.parse_source == "llm_parse_failed"


def test_early_missing_score_and_unparsed_flag() -> None:
    missing_score = _GeneratedJudge(['{"flag":"Neutral","rationale":"ok"}'])
    early = missing_score.judge_early({"question_text": "q", "gt_answer": "a"}, "ctx", "hi")
    assert early.category == "unparsed"
    assert early.score == 0.0
    assert early.parse_source == "llm_parse_failed"

    garbage = _GeneratedJudge(["not json and not a known flag"])
    unparsed = garbage.judge_early({"question_text": "q", "gt_answer": "a"}, "ctx", "hi")
    assert unparsed.category == "unparsed"
    assert unparsed.score == 0.0
    assert unparsed.parse_source == "llm_parse_failed"


@pytest.mark.parametrize(
    ("path", "response"),
    [
        ("early", '{"flag":"Neutral","rationale":"Could not evaluate this answer"}'),
        ("early", '{"flag":"Neutral","score":"invalid","rationale":"bad"}'),
        ("core", '{"rationale":"Could not evaluate this answer"}'),
        ("core", '{"rationale":"Cannot score: 1 required reference is missing."}'),
        ("core", '{"score":"invalid","trigger_phrase":"","spoiler":false,"rationale":"bad"}'),
        ("partial", '{"rationale":"Could not evaluate this answer"}'),
        ("partial", '{"score":"invalid","hallucination":false,"rationale":"bad"}'),
    ],
)
def test_missing_or_invalid_json_scores_are_parse_failures(path: str, response: str) -> None:
    slot = {"question_text": "q", "gt_answer": "a"}
    judge = _GeneratedJudge([response])
    if path == "early":
        judgment = judge.judge_early(slot, "ctx", "hi")
    elif path == "core":
        judgment = judge.judge_core(slot, "ctx", "answer", "(none)")
    else:
        judgment = judge.judge_interrupted_partial(slot, "answer")
    assert judgment.score == 0.0
    assert judgment.parse_source == "llm_parse_failed"


def test_explicit_zero_json_score_is_valid_llm_json() -> None:
    slot = {"question_text": "q", "gt_answer": "a"}
    early = _GeneratedJudge(['{"flag":"Neutral","score":0,"rationale":"zero"}']).judge_early(slot, "ctx", "hi")
    core = _GeneratedJudge(['{"score":0.0,"trigger_phrase":"","spoiler":false,"rationale":"zero"}']).judge_core(
        slot, "ctx", "answer", "(none)"
    )
    partial = _GeneratedJudge(['{"score":0,"hallucination":false,"rationale":"zero"}']).judge_interrupted_partial(
        slot, "answer"
    )
    assert early.category == "neutral"
    assert early.score == 0.0
    assert early.parse_source == "llm_json"
    assert core.score == 0.0
    assert core.parse_source == "llm_json"
    assert partial.score == 0.0
    assert partial.parse_source == "llm_json"


@pytest.mark.parametrize("score_literal", ['"NaN"', '"Infinity"', '"-Infinity"', "NaN", "Infinity", "-Infinity"])
def test_nonfinite_json_scores_are_parse_failures(score_literal: str) -> None:
    slot = {"question_text": "q", "gt_answer": "a"}
    early = _GeneratedJudge([f'{{"flag":"Neutral","score":{score_literal},"rationale":"bad"}}']).judge_early(
        slot, "ctx", "hi"
    )
    core = _GeneratedJudge(
        [f'{{"score":{score_literal},"trigger_phrase":"answer","spoiler":false,"rationale":"bad"}}']
    ).judge_core(slot, "ctx", "answer", "(none)")
    partial = _GeneratedJudge(
        [f'{{"score":{score_literal},"hallucination":false,"rationale":"bad"}}']
    ).judge_interrupted_partial(slot, "answer")

    assert early.score == 0.0
    assert early.parse_source == "llm_parse_failed"
    assert core.score == 0.0
    assert core.parse_source == "llm_parse_failed"
    assert core.trigger_phrase == ""
    assert partial.score == 0.0
    assert partial.parse_source == "llm_parse_failed"


def test_fingerprint_requires_evaluator_schema_version() -> None:
    from vllm_omni.benchmarks.omniinteract_eval import EvaluationInputsFingerprint

    payload = {
        "annotation_sha256": "a" * 64,
        "transcript_sha256": "b" * 64,
        "judge_model": "m",
        "judge_base_url": "http://127.0.0.1:9",
        "judge_max_tokens": 32,
        "protocol_source": "proto",
    }
    assert EvaluationInputsFingerprint.from_mapping(payload) is None
    payload["evaluator_schema_version"] = 1
    assert EvaluationInputsFingerprint.from_mapping(payload) is not None
    payload["evaluator_schema_version"] = 999
    matched = EvaluationInputsFingerprint.from_mapping(payload)
    assert matched is not None
    assert matched.evaluator_schema_version == 999


def test_summarize_counts_parse_sources() -> None:
    rows = [
        {
            **_slot_row(scene_type="multi_turn", question_type="realtime", tp_n=1.0),
            "stage_early": {"parse_source": "llm_json"},
            "stage_core": {"parse_source": "llm_parse_failed", "trigger_fallback": True},
        }
    ]
    summary = _summarize(rows, unmatched=0)
    assert summary["judge_parse"]["llm_json"] == 1
    assert summary["judge_parse"]["llm_parse_failed"] == 1
    assert summary["judge_parse"]["judge_calls"] == 2
    assert summary["trigger_fallback_slots"] == 1