91 lines
3.7 KiB
Python
91 lines
3.7 KiB
Python
"""Numerical parity: HF reference Spark-X2.5 vs the vLLM-Omni implementation.
|
|||
|
|
|
||
|
|
Runs both on CPU, greedy, and compares generated ids plus the first-step
|
||
|
|
logit vector.
|
||
|
|
"""
|
||
|
|
import argparse, json, os, sys
|
||
|
|
import numpy as np
|
||
|
|
import torch
|
||
|
|
|
||
|
|
MODEL = os.environ.get("SPARK_MODEL", "XHToken/Spark-X2.5-1.7B")
|
||
|
|
PROMPT_MSGS = [
|
||
|
|
{"role": "user", "content": "What is the capital of France? Answer in one word."},
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def hf_reference(model_path, prompt_ids, n_new, dtype):
|
||
|
|
from transformers import AutoConfig, AutoModelForCausalLM
|
||
|
|
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||
|
|
|
||
|
|
# modeling_spark.py targets transformers 4.57; v5 expects the tied-weight
|
||
|
|
# map to be a dict rather than a list of keys.
|
||
|
|
cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
|
||
|
|
cls = get_class_from_dynamic_module(
|
||
|
|
cfg.auto_map["AutoModelForCausalLM"], model_path
|
||
|
|
)
|
||
|
|
if isinstance(getattr(cls, "_tied_weights_keys", None), list):
|
||
|
|
cls._tied_weights_keys = {"lm_head.weight": "model.embedding.weight"}
|
||
|
|
m = cls.from_pretrained(model_path, dtype=dtype).eval()
|
||
|
|
ids = torch.tensor([prompt_ids])
|
||
|
|
with torch.no_grad():
|
||
|
|
out = m(ids)
|
||
|
|
first_logits = out.logits[0, -1].float().numpy()
|
||
|
|
gen = m.generate(ids, max_new_tokens=n_new, do_sample=False,
|
||
|
|
pad_token_id=2)
|
||
|
|
return first_logits, gen[0, len(prompt_ids):].tolist()
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
ap.add_argument("--n-new", type=int, default=24)
|
||
|
|
ap.add_argument("--dtype", default="bfloat16")
|
||
|
|
ap.add_argument("--max-model-len", type=int, default=2048)
|
||
|
|
ap.add_argument("--out", default="parity.json")
|
||
|
|
args = ap.parse_args()
|
||
|
|
dtype = getattr(torch, args.dtype)
|
||
|
|
|
||
|
|
from transformers import AutoTokenizer
|
||
|
|
tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
|
||
|
|
prompt = tok.apply_chat_template(PROMPT_MSGS, tokenize=False,
|
||
|
|
add_generation_prompt=True)
|
||
|
|
prompt_ids = tok(prompt, add_special_tokens=False)["input_ids"]
|
||
|
|
print(f"prompt tokens: {len(prompt_ids)}")
|
||
|
|
|
||
|
|
hf_logits, hf_ids = hf_reference(MODEL, prompt_ids, args.n_new, dtype)
|
||
|
|
print("HF :", repr(tok.decode(hf_ids)))
|
||
|
|
|
||
|
|
import vllm_omni # noqa: F401 (registers Spark2_5ForCausalLM)
|
||
|
|
from vllm import LLM, SamplingParams
|
||
|
|
llm = LLM(model=MODEL, dtype=args.dtype, max_model_len=args.max_model_len,
|
||
|
|
enforce_eager=True, gpu_memory_utilization=0.30,
|
||
|
|
max_num_seqs=1, trust_remote_code=False)
|
||
|
|
sp = SamplingParams(temperature=0.0, max_tokens=args.n_new, logprobs=0,
|
||
|
|
prompt_logprobs=0)
|
||
|
|
res = llm.generate([{"prompt_token_ids": prompt_ids}], sp)[0]
|
||
|
|
v_ids = list(res.outputs[0].token_ids)
|
||
|
|
print("vLLM:", repr(res.outputs[0].text))
|
||
|
|
|
||
|
|
n = min(len(hf_ids), len(v_ids))
|
||
|
|
match = sum(int(a == b) for a, b in zip(hf_ids[:n], v_ids[:n]))
|
||
|
|
# first generated token's logit agreement
|
||
|
|
first_hf_top = int(np.argmax(hf_logits))
|
||
|
|
report = {
|
||
|
|
"prompt_len": len(prompt_ids),
|
||
|
|
"dtype": args.dtype,
|
||
|
|
"hf_text": tok.decode(hf_ids),
|
||
|
|
"vllm_text": res.outputs[0].text,
|
||
|
|
"hf_ids": hf_ids, "vllm_ids": v_ids,
|
||
|
|
"greedy_token_match": f"{match}/{n}",
|
||
|
|
"exact_match": hf_ids[:n] == v_ids[:n],
|
||
|
|
"hf_first_top1": first_hf_top,
|
||
|
|
"vllm_first_token": v_ids[0] if v_ids else None,
|
||
|
|
"first_token_agree": bool(v_ids and v_ids[0] == first_hf_top),
|
||
|
|
}
|
||
|
|
json.dump(report, open(args.out, "w"), indent=2)
|
||
|
|
print(json.dumps({k: v for k, v in report.items()
|
||
|
|
if k not in ("hf_ids", "vllm_ids")}, indent=2))
|
||
|
|
return 0 if report["exact_match"] else 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|