Files
vllm-omni/benchmarks/edge_harness/experiments/parity_hf_vs_vllm.py
T

91 lines
3.7 KiB
Python
Raw Normal View History

"""Numerical parity: HF reference Spark-X2.5 vs the vLLM-Omni implementation.
Runs both on CPU, greedy, and compares generated ids plus the first-step
logit vector.
"""
import argparse, json, os, sys
import numpy as np
import torch
MODEL = os.environ.get("SPARK_MODEL", "XHToken/Spark-X2.5-1.7B")
PROMPT_MSGS = [
{"role": "user", "content": "What is the capital of France? Answer in one word."},
]
def hf_reference(model_path, prompt_ids, n_new, dtype):
from transformers import AutoConfig, AutoModelForCausalLM
from transformers.dynamic_module_utils import get_class_from_dynamic_module
# modeling_spark.py targets transformers 4.57; v5 expects the tied-weight
# map to be a dict rather than a list of keys.
cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
cls = get_class_from_dynamic_module(
cfg.auto_map["AutoModelForCausalLM"], model_path
)
if isinstance(getattr(cls, "_tied_weights_keys", None), list):
cls._tied_weights_keys = {"lm_head.weight": "model.embedding.weight"}
m = cls.from_pretrained(model_path, dtype=dtype).eval()
ids = torch.tensor([prompt_ids])
with torch.no_grad():
out = m(ids)
first_logits = out.logits[0, -1].float().numpy()
gen = m.generate(ids, max_new_tokens=n_new, do_sample=False,
pad_token_id=2)
return first_logits, gen[0, len(prompt_ids):].tolist()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--n-new", type=int, default=24)
ap.add_argument("--dtype", default="bfloat16")
ap.add_argument("--max-model-len", type=int, default=2048)
ap.add_argument("--out", default="parity.json")
args = ap.parse_args()
dtype = getattr(torch, args.dtype)
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
prompt = tok.apply_chat_template(PROMPT_MSGS, tokenize=False,
add_generation_prompt=True)
prompt_ids = tok(prompt, add_special_tokens=False)["input_ids"]
print(f"prompt tokens: {len(prompt_ids)}")
hf_logits, hf_ids = hf_reference(MODEL, prompt_ids, args.n_new, dtype)
print("HF :", repr(tok.decode(hf_ids)))
import vllm_omni # noqa: F401 (registers Spark2_5ForCausalLM)
from vllm import LLM, SamplingParams
llm = LLM(model=MODEL, dtype=args.dtype, max_model_len=args.max_model_len,
enforce_eager=True, gpu_memory_utilization=0.30,
max_num_seqs=1, trust_remote_code=False)
sp = SamplingParams(temperature=0.0, max_tokens=args.n_new, logprobs=0,
prompt_logprobs=0)
res = llm.generate([{"prompt_token_ids": prompt_ids}], sp)[0]
v_ids = list(res.outputs[0].token_ids)
print("vLLM:", repr(res.outputs[0].text))
n = min(len(hf_ids), len(v_ids))
match = sum(int(a == b) for a, b in zip(hf_ids[:n], v_ids[:n]))
# first generated token's logit agreement
first_hf_top = int(np.argmax(hf_logits))
report = {
"prompt_len": len(prompt_ids),
"dtype": args.dtype,
"hf_text": tok.decode(hf_ids),
"vllm_text": res.outputs[0].text,
"hf_ids": hf_ids, "vllm_ids": v_ids,
"greedy_token_match": f"{match}/{n}",
"exact_match": hf_ids[:n] == v_ids[:n],
"hf_first_top1": first_hf_top,
"vllm_first_token": v_ids[0] if v_ids else None,
"first_token_agree": bool(v_ids and v_ids[0] == first_hf_top),
}
json.dump(report, open(args.out, "w"), indent=2)
print(json.dumps({k: v for k, v in report.items()
if k not in ("hf_ids", "vllm_ids")}, indent=2))
return 0 if report["exact_match"] else 1
if __name__ == "__main__":
sys.exit(main())