# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

"""
Unit tests for metrics.py
"""

import math

import pytest
from vllm.benchmarks.serve import TaskType

from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput

pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]


def test_tpot_matches_mean_itl_per_request():
    """TPOT and pooled ITL should agree when output_len tracks ITL samples."""
    output = MixRequestFuncOutput()
    output.success = True
    output.prompt_len = 100
    # Simulate server reporting more tokens than SSE chunks (bundled tokens).
    # len(itl)=2 → 2 inter-chunk intervals, but server generated 10 tokens.
    output.output_tokens = 10
    output.generated_text = "hello world"
    output.ttft = 0.05
    output.text_latency = 0.25
    output.latency = 0.30
    output.itl = [0.10, 0.10]

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=[output],
        dur_s=1.0,
        tokenizer=None,
        selected_percentiles=[50.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=["tpot", "itl"],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=1.0,
    )

    assert metrics.mean_tpot_ms == pytest.approx(metrics.mean_itl_ms, rel=1e-6, abs=1e-6)


def _make_output(prompt_len: int, output_tokens: int = 10) -> MixRequestFuncOutput:
    """Build a minimal successful MixRequestFuncOutput for metrics aggregation."""
    output = MixRequestFuncOutput()
    output.success = True
    output.prompt_len = prompt_len
    output.output_tokens = output_tokens
    output.generated_text = "x" * output_tokens
    output.ttft = 0.1
    output.text_latency = 1.0
    output.latency = 1.0
    output.start_time = 0.0
    output.itl = [0.1] * max(output_tokens - 1, 0)
    output.audio_ttfp = 0.0
    output.audio_rtf = 0.0
    output.audio_duration = 0.0
    output.audio_frames = 0
    output.input_audio_duration = 0.0
    output.error = ""
    return output


def _calculate_test_metrics(outputs, goodput=None):
    return calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=1.0,
        tokenizer=None,
        selected_percentiles=[50.0],
        goodput_config_dict=goodput or {},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=[],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=1.0,
    )[0]


# ============================================================================
# total_input Tests
# ============================================================================


def test_total_input_aggregated_from_output_prompt_len():
    """Test that total_input sums outputs[i].prompt_len, not input_requests[i].prompt_len."""
    outputs = [_make_output(4992), _make_output(3000)]

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=10.0,
        tokenizer=None,
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=[],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=10.0,
    )

    assert metrics.total_input == 7992, (
        "total_input should aggregate from outputs[i].prompt_len to reflect the true multimodal input token count"
    )


def test_audio_continuity_aggregation():
    """Continuity rate and underrun percentile must aggregate from per-output fields."""
    bad = _make_output(100)
    bad.audio_underrun_s = 0.5
    bad.audio_continuity_ok = False
    good_a = _make_output(100)
    good_a.audio_underrun_s = 0.02
    good_a.audio_continuity_ok = True
    good_b = _make_output(100)
    good_b.audio_underrun_s = 0.0
    good_b.audio_continuity_ok = True

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=[bad, good_a, good_b],
        dur_s=10.0,
        tokenizer=None,
        selected_percentiles=[50.0, 99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=["audio_underrun"],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=10.0,
    )

    assert metrics.audio_continuity_ok_rate == pytest.approx(2 / 3, abs=1e-6)
    # p99 of [0.5, 0.02, 0.0] is dominated by the 0.5 outlier.
    p99 = dict(metrics.percentiles_audio_underrun_s).get(99.0)
    assert p99 is not None and p99 > 0.4


def test_unmeasured_duplex_latency_does_not_add_zero_samples():
    measured = _make_output(100, output_tokens=2)
    measured.ttft = 0.1
    measured.audio_ttfp = 0.2
    measured.audio_rtf = 0.5
    measured.duplex_session_metrics = {
        "ttft_ms": 100.0,
        "ttfp_ms": 200.0,
        "rtf": 0.5,
    }
    listen_only = _make_output(100, output_tokens=0)
    listen_only.ttft = listen_only.audio_ttfp = listen_only.audio_rtf = 0.0
    listen_only.duplex_session_metrics = {
        "ttft_ms": None,
        "ttfp_ms": None,
        "rtf": None,
    }

    metrics = _calculate_test_metrics([measured, listen_only], {"ttft": 150.0})

    assert metrics.mean_ttft_ms == 100.0
    assert metrics.mean_audio_ttfp_ms == 200.0
    assert metrics.mean_audio_rtf == 0.5
    assert (metrics.num_ttft_samples, metrics.num_audio_ttfp_samples, metrics.num_audio_rtf_samples) == (1, 1, 1)
    assert metrics.request_goodput == 1.0


def test_all_unmeasured_duplex_latency_is_not_reported_as_zero():
    listen_only = _make_output(100, output_tokens=0)
    listen_only.duplex_session_metrics = {
        "ttft_ms": None,
        "ttfp_ms": None,
        "rtf": None,
    }

    metrics = _calculate_test_metrics(
        [listen_only],
        {"ttft": float("inf"), "audio_ttft": float("inf")},
    )

    assert (metrics.num_ttft_samples, metrics.num_audio_ttfp_samples, metrics.num_audio_rtf_samples) == (0, 0, 0)
    assert math.isnan(metrics.mean_ttft_ms)
    assert math.isnan(metrics.mean_audio_ttfp_ms)
    assert math.isnan(metrics.mean_audio_rtf)
    assert metrics.request_goodput == 0.0


def test_unmeasured_duplex_tpot_does_not_add_zero_or_misalign_goodput():
    missing_tpot = _make_output(100, output_tokens=5)
    missing_tpot.itl = []
    missing_tpot.text_latency = missing_tpot.ttft = 0.1
    missing_tpot.tpot_measured = False
    missing_tpot.duplex_session_metrics = {"ttft_ms": 100.0}

    slow_ttft = _make_output(100, output_tokens=5)
    slow_ttft.ttft = 1.0
    slow_ttft.text_latency = 1.4
    slow_ttft.itl = [0.1] * 4
    slow_ttft.duplex_session_metrics = {"ttft_ms": 1000.0}

    metrics = _calculate_test_metrics(
        [missing_tpot, slow_ttft],
        {"ttft": 500.0, "tpot": 200.0},
    )

    assert metrics.num_tpot_samples == 1
    assert metrics.mean_tpot_ms == pytest.approx(100.0)
    assert metrics.request_goodput == 0.0


def test_all_unmeasured_duplex_token_timing_is_not_reported_as_zero():
    output = _make_output(100, output_tokens=5)
    output.itl = []
    output.text_latency = output.ttft
    output.tpot_measured = False
    output.duplex_session_metrics = {"ttft_ms": 100.0}

    metrics = _calculate_test_metrics([output], {"tpot": float("inf")})

    assert (metrics.num_tpot_samples, metrics.num_itl_samples) == (0, 0)
    assert math.isnan(metrics.mean_tpot_ms)
    assert math.isnan(metrics.mean_itl_ms)
    assert metrics.request_goodput == 0.0


def test_duplex_response_timings_do_not_build_a_session_token_timeline():
    output = _make_output(100, output_tokens=5)
    output.latency = 101.0
    output.duplex_request_metrics = [
        {"response_id": "r1", "stage0_tokens": {"itls_ms": [100.0, 100.0]}},
        {"response_id": "r2", "stage0_tokens": {"itls_ms": [100.0, 100.0]}},
    ]
    output.duplex_session_metrics = {"ttft_ms": 100.0}

    metrics = _calculate_test_metrics([output])

    assert math.isnan(metrics.max_output_tokens_per_s)
    assert metrics.max_concurrent_requests == 1
    assert metrics.mean_tpot_ms == pytest.approx(100.0)
    assert metrics.mean_itl_ms == pytest.approx(100.0)


def test_unmeasured_tpot_stays_missing_after_tokenizer_fallback():
    output = _make_output(100, output_tokens=0)
    output.generated_text = "timing metadata missing"
    output.itl = []
    output.text_latency = output.ttft
    output.tpot_measured = False
    output.duplex_session_metrics = {"ttft_ms": 100.0}

    def tokenizer(text, *, add_special_tokens):
        assert text == output.generated_text
        assert add_special_tokens is False
        return type("Tokenized", (), {"input_ids": [1, 2, 3]})()

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=[output],
        dur_s=1.0,
        tokenizer=tokenizer,
        selected_percentiles=[50.0],
        goodput_config_dict={"tpot": float("inf")},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=[],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=1.0,
    )

    assert metrics.total_output == 3
    assert metrics.num_tpot_samples == 0
    assert math.isnan(metrics.mean_tpot_ms)
    assert metrics.request_goodput == 0.0


def test_zero_itl_does_not_create_zero_tpot():
    output = _make_output(100, output_tokens=3)
    output.itl = [0.0, 0.0]
    output.duplex_session_metrics = {"ttft_ms": 100.0}

    metrics = _calculate_test_metrics([output], {"tpot": 1.0})

    assert (metrics.num_tpot_samples, metrics.num_itl_samples) == (0, 2)
    assert math.isnan(metrics.mean_tpot_ms)
    assert metrics.mean_itl_ms == 0.0
    assert metrics.request_goodput == 0.0


def test_stage_output_tokens_without_client_timing_omit_tpot(capsys):
    output = _make_output(100, output_tokens=5)
    output.itl = []
    output.text_latency = 0.0
    output.ttft = 0.1

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=[output],
        dur_s=1.0,
        tokenizer=None,
        selected_percentiles=[50.0, 99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=["tpot"],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=1.0,
    )

    printed = capsys.readouterr().out
    assert metrics.num_tpot_samples == 0
    assert math.isnan(metrics.mean_tpot_ms)
    assert "Time per Output Token" not in printed
    assert "Mean TPOT" not in printed


def test_consistent_client_latency_can_supply_tpot_fallback():
    output = _make_output(100, output_tokens=10)
    output.itl = []
    output.ttft = 0.1
    output.text_latency = 1.0

    metrics = _calculate_test_metrics([output])

    assert metrics.num_tpot_samples == 1
    assert metrics.mean_tpot_ms == pytest.approx(100.0)


def test_single_token_responses_do_not_report_zero_tpot(capsys):
    output = _make_output(100, output_tokens=1)

    metrics, _ = calculate_metrics(
        input_requests=[],
        outputs=[output],
        dur_s=1.0,
        tokenizer=None,
        selected_percentiles=[50.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=["tpot"],
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=1.0,
    )

    printed = capsys.readouterr().out
    assert metrics.num_tpot_samples == 0
    assert math.isnan(metrics.mean_tpot_ms)
    assert "Time per Output Token" not in printed


def test_duplex_goodput_does_not_pair_measurements_from_different_requests():
    text_only, audio_only = _make_output(100), _make_output(100)
    text_only.ttft, text_only.audio_ttfp = 0.1, 0.0
    text_only.duplex_session_metrics = {"ttft_ms": 100.0, "ttfp_ms": None, "rtf": None}
    audio_only.ttft, audio_only.audio_ttfp = 0.0, 0.2
    audio_only.duplex_session_metrics = {"ttft_ms": None, "ttfp_ms": 200.0, "rtf": None}

    metrics = _calculate_test_metrics([text_only, audio_only], {"ttft": 500.0, "audio_ttft": 500.0})

    assert metrics.request_goodput == 0.0


# ============================================================================
# TTFT suppression for pure-audio (TTS) benchmarks
# ============================================================================


class _EmptyAwareTokenizer:
    """Minimal tokenizer stub: token count == len(text), so '' -> 0 tokens.

    Mirrors production where a TTS speech endpoint returns empty generated_text,
    making total_output == 0 (the real CI path uses a real tokenizer, not None).
    """

    def __call__(self, text, add_special_tokens=False):
        class _R:
            pass

        r = _R()
        r.input_ids = [0] * len(text)
        return r


def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
    """Pure-TTS output: no text tokens, only audio. ttft is left unset (0.0)."""
    output = MixRequestFuncOutput()
    output.success = True
    output.prompt_len = prompt_len
    output.output_tokens = 0
    output.generated_text = ""
    output.ttft = 0.0
    output.text_latency = 1.0
    output.latency = 1.0
    output.start_time = 0.0
    output.itl = []
    output.audio_ttfp = 0.05
    output.audio_rtf = 0.2
    output.audio_duration = 5.0
    output.audio_frames = 120000
    output.input_audio_duration = 0.0
    output.error = ""
    return output


_TTS_PERCENTILE_METRICS = ["ttft", "e2el", "audio_rtf", "audio_ttfp", "audio_duration"]


def test_tts_benchmark_omits_ttft(capsys):
    """Pure-TTS run (total_output == 0) must not print a Time to First Token section."""
    outputs = [_make_tts_output(100), _make_tts_output(120)]

    calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=10.0,
        tokenizer=_EmptyAwareTokenizer(),
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=10.0,
    )

    out = capsys.readouterr().out
    assert "Time to First Token" not in out, "TTS bench must not surface a meaningless TTFT"
    assert "Time to First Packet" in out, "audio TTFP must still be reported"
    assert "End-to-end Latency" in out, "e2el must still be reported"


def test_text_benchmark_still_reports_ttft(capsys):
    """Regression guard: real text generation (total_output > 0) keeps TTFT."""
    outputs = [_make_output(100, output_tokens=10), _make_output(120, output_tokens=10)]

    calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=10.0,
        tokenizer=_EmptyAwareTokenizer(),
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=10.0,
    )

    out = capsys.readouterr().out
    assert "Time to First Token" in out, "text benchmarks must keep TTFT"


# ============================================================================
# Suppress text metrics for pure image / video endpoints
# ============================================================================


def _make_image_output(prompt_len: int) -> MixRequestFuncOutput:
    output = MixRequestFuncOutput()
    output.success = True
    output.prompt_len = prompt_len
    output.output_tokens = 0
    output.generated_text = ""
    output.ttft = 0.0
    output.text_latency = 0.0
    output.latency = 2.5
    output.start_time = 0.0
    output.itl = []
    output.image_count = 1
    output.image_generation_time_ms = 2000.0
    output.image_pixels = 1024 * 1024
    output.input_audio_duration = 0.0
    output.error = ""
    return output


def _make_video_output(prompt_len: int) -> MixRequestFuncOutput:
    output = MixRequestFuncOutput()
    output.success = True
    output.prompt_len = prompt_len
    output.output_tokens = 0
    output.generated_text = ""
    output.ttft = 0.0
    output.text_latency = 0.0
    output.latency = 5.0
    output.start_time = 0.0
    output.itl = []
    output.video_duration = 2.0
    output.video_frames = 48
    output.video_generation_time_ms = 4000.0
    output.video_rtf = 2.0
    output.input_audio_duration = 0.0
    output.error = ""
    return output


_MEDIA_PERCENTILE_METRICS = ["e2el", "ttft"]


def test_pure_image_benchmark_omits_text_result(capsys):
    """Pure image generation must not print Text Result or invent peak tok/s."""
    outputs = [_make_image_output(64), _make_image_output(64)]

    metrics, actual_output_lens = calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=5.0,
        tokenizer=None,
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=5.0,
    )

    out = capsys.readouterr().out
    assert actual_output_lens == [0, 0]
    assert metrics.total_output == 0
    assert metrics.max_output_tokens_per_s == 0.0
    assert " Text Result " not in out
    assert "Peak output token throughput" not in out
    assert "Time to First Token" not in out
    assert " Image Result " in out


def test_pure_video_benchmark_omits_text_result(capsys):
    """Pure video generation must not print Text Result or invent peak tok/s."""
    outputs = [_make_video_output(63), _make_video_output(63)]

    metrics, actual_output_lens = calculate_metrics(
        input_requests=[],
        outputs=outputs,
        dur_s=10.0,
        tokenizer=None,
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=10.0,
    )

    out = capsys.readouterr().out
    assert actual_output_lens == [0, 0]
    assert metrics.total_output == 0
    assert metrics.max_output_tokens_per_s == 0.0
    assert " Text Result " not in out
    assert "Peak output token throughput" not in out
    assert "Time to First Token" not in out
    assert " Video Result " in out


def test_image_with_generated_text_still_reports_text_result(capsys):
    """Image edits / AR text alongside images must keep Text Result."""
    output = _make_image_output(64)
    output.output_tokens = 8
    output.generated_text = "revised caption"
    output.ttft = 0.05
    output.itl = [0.01] * 7

    calculate_metrics(
        input_requests=[],
        outputs=[output],
        dur_s=3.0,
        tokenizer=_EmptyAwareTokenizer(),
        selected_percentiles=[99.0],
        goodput_config_dict={},
        task_type=TaskType.GENERATION,
        selected_percentile_metrics=_MEDIA_PERCENTILE_METRICS,
        max_concurrency=None,
        request_rate=float("inf"),
        benchmark_duration=3.0,
    )

    out = capsys.readouterr().out
    assert " Text Result " in out
    assert "Time to First Token" in out
    assert " Image Result " in out


if __name__ == "__main__":
    pytest.main([__file__, "-v", "-s"])