{"tag": "vllm-bf16-t32", "threads": "32", "gen": 128, "prompt": 128, "tok_s_by_batch": {"1": 34.56, "2": 69.87, "4": 139.76, "8": 266.13, "16": 495.67}}
{"tag": "vllm-int8-t32", "threads": "32", "gen": 128, "prompt": 128, "tok_s_by_batch": {"1": 54.87, "2": 105.21, "4": 210.3, "8": 387.84, "16": 694.7}}
{"tag": "vllm-int8-t32-hi", "threads": "32", "gen": 128, "prompt": 128, "tok_s_by_batch": {"32": 1080.59, "64": 1557.9}}