Files
vllm-omni/benchmarks/edge_harness/fidelity_kl.py
T

179 lines
7.7 KiB
Python
Raw Normal View History

"""A fidelity metric that does not saturate.
Greedy divergence over 12 prompts -- "did the quantized build reproduce the
bf16 output exactly?" -- answers 2/12 for W4A16 and 1/12 for W4A8 and cannot
say whether the gap between them is small or enormous. Everything past the
first divergent token is thrown away, and one unlucky token near position 3
scores the same as a build that is wrong everywhere.
This measures the distribution instead, teacher-forced over a corpus, so every
position contributes:
* **top-1 agreement** -- how often the quantized model's argmax matches the
reference's. This is what actually decides whether greedy decoding diverges.
* **KL(reference || candidate)** -- the same quantity ``llama-perplexity
--kl-divergence`` reports, so llama.cpp's Q4_K_M stays comparable against
our builds rather than being compared on a different metric.
* **Delta log-prob of the reference token**, which is the cross-entropy
difference and is what perplexity is built from.
One honest limitation, stated because it changes how the number should be
read: vLLM returns the top-k log-probs per position, not the full
distribution, so KL is computed over the union of the two top-k sets with the
remaining mass as a single tail bucket. ``coverage`` reports the reference
probability mass that fell inside the top-k, and a run whose coverage is low
is reporting a lower bound, not a KL. llama.cpp stores full logits and has no
such limitation; raise ``--topk`` (and vLLM's ``--max-logprobs``) to close the
gap when the two are compared directly.
"""
from __future__ import annotations
import argparse, json, math, os, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from bench_harness import provenance # noqa: E402
def chunk_tokens(token_ids: list[int], size: int, limit: int | None) -> list[list[int]]:
chunks = [token_ids[i:i + size] for i in range(0, len(token_ids), size)]
chunks = [c for c in chunks if len(c) >= 16]
return chunks[:limit] if limit else chunks
def collect(llm, chunks, topk: int):
"""Teacher-forced top-k log-probs at every position of every chunk."""
from vllm import SamplingParams
sp = SamplingParams(temperature=0.0, max_tokens=1, prompt_logprobs=topk)
outs = llm.generate([{"prompt_token_ids": c} for c in chunks], sp,
use_tqdm=False)
positions = []
for out, chunk in zip(outs, chunks):
for i, entry in enumerate(out.prompt_logprobs or []):
if entry is None: # first token has no prediction
continue
positions.append({
"target": chunk[i],
"logprobs": {int(tid): float(lp.logprob) for tid, lp in entry.items()},
})
return positions
def _top1(d: dict[int, float]) -> int:
return max(d.items(), key=lambda kv: kv[1])[0]
def compare(ref: list[dict], cand: list[dict]) -> dict:
if len(ref) != len(cand):
raise SystemExit(f"position count differs: {len(ref)} vs {len(cand)}; "
f"the two runs did not see the same corpus")
n = len(ref)
agree = 0
kls, dlps, coverages = [], [], []
for r, c in zip(ref, cand):
rl, cl = r["logprobs"], c["logprobs"]
if _top1(rl) == _top1(cl):
agree += 1
# Reference distribution over its own top-k, plus a tail bucket.
rp = {t: math.exp(lp) for t, lp in rl.items()}
covered = sum(rp.values())
coverages.append(min(covered, 1.0))
tail_r = max(0.0, 1.0 - covered)
# Candidate probability for the same tokens. A token outside the
# candidate's top-k is bounded by its smallest returned probability,
# which overestimates q and therefore underestimates KL -- the reason
# this is reported as a lower bound when coverage is poor.
floor = math.exp(min(cl.values())) if cl else 1e-12
cq = {t: math.exp(cl[t]) if t in cl else floor for t in rp}
covered_c = sum(cq.values())
tail_c = max(1e-12, 1.0 - min(covered_c, 1.0 - 1e-12))
kl = sum(p * math.log(p / max(cq[t], 1e-12)) for t, p in rp.items() if p > 0)
if tail_r > 0:
kl += tail_r * math.log(tail_r / tail_c)
kls.append(max(kl, 0.0))
target = r["target"]
if target in rl and target in cl:
dlps.append(cl[target] - rl[target])
kls.sort()
return {
"positions": n,
"top1_agreement": round(agree / n, 4) if n else None,
"kl_mean": round(sum(kls) / len(kls), 6) if kls else None,
"kl_median": round(kls[len(kls) // 2], 6) if kls else None,
"kl_p99": round(kls[int(len(kls) * 0.99)], 6) if kls else None,
"delta_logprob_mean": round(sum(dlps) / len(dlps), 6) if dlps else None,
"delta_logprob_n": len(dlps),
"coverage_mean": round(sum(coverages) / len(coverages), 4) if coverages else None,
}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--model", required=True)
ap.add_argument("--quantization", default=None)
ap.add_argument("--corpus", required=True, help="UTF-8 text file")
ap.add_argument("--tokenizer", default=None,
help="tokenizer to chunk with; defaults to --model")
ap.add_argument("--chunk", type=int, default=512)
ap.add_argument("--max-chunks", type=int, default=8)
ap.add_argument("--topk", type=int, default=64)
ap.add_argument("--max-model-len", type=int, default=2048)
ap.add_argument("--save-ref", default=None,
help="write this run's distributions as the reference")
ap.add_argument("--ref", default=None,
help="reference file to compare against")
ap.add_argument("--label", default="")
ap.add_argument("--out", required=True)
a = ap.parse_args()
if not a.save_ref and not a.ref:
ap.error("give --save-ref (to record a reference) or --ref (to compare)")
import vllm_omni # noqa: F401
from transformers import AutoTokenizer
from vllm import LLM
tok = AutoTokenizer.from_pretrained(a.tokenizer or a.model,
trust_remote_code=False)
text = open(a.corpus, encoding="utf-8").read()
chunks = chunk_tokens(tok(text).input_ids, a.chunk, a.max_chunks)
if not chunks:
raise SystemExit("corpus too short for one chunk")
llm = LLM(model=a.model, dtype="bfloat16", max_model_len=a.max_model_len,
max_num_seqs=1, quantization=a.quantization,
enable_prefix_caching=False, trust_remote_code=False,
disable_log_stats=True, max_logprobs=a.topk)
positions = collect(llm, chunks, a.topk)
meta = {"label": a.label, "model": a.model, "quantization": a.quantization,
"corpus": os.path.abspath(a.corpus), "chunk": a.chunk,
"n_chunks": len(chunks), "topk": a.topk,
"provenance": provenance({"metric": "fidelity_kl"})}
if a.save_ref:
with open(a.save_ref, "w") as f:
json.dump({**meta, "positions": positions}, f)
print(f"wrote reference: {len(positions)} positions -> {a.save_ref}")
result = {**meta, "role": "reference", "n_positions": len(positions)}
else:
ref = json.loads(open(a.ref).read())
if ref["chunk"] != a.chunk or ref["n_chunks"] != len(chunks):
raise SystemExit("reference was taken on a different corpus slicing")
result = {**meta, "role": "candidate",
"reference_model": ref["model"],
**compare(ref["positions"], positions)}
print(json.dumps({k: v for k, v in result.items()
if k != "provenance"}, indent=2))
with open(a.out, "w") as f:
json.dump(result, f, indent=2)
if __name__ == "__main__":
main()