{"tag": "base", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-int4-g64", "dtype": "bfloat16", "compilation_config": null, "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 83.15, "rss_mb": 8663.0, "rss_peak_mb": 8699.4, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.2227, "pp_tps": 2299.08, "tg_tokens": 128, "tg_s": 2.2158, "tg_tps": 57.32, "raw": {"pp": [0.31149862613528967, 0.3030338645912707, 0.22269816184416413], "tg": [2.3680049385875463, 2.3106129062362015, 2.229406553786248], "tg1": [0.01362658105790615, 0.013699103146791458, 0.013955966103821993]}} {"tag": "gate", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-int4-g64", "dtype": "bfloat16", "compilation_config": null, "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 50.46, "rss_mb": 8681.6, "rss_peak_mb": 8710.0, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.1508, "pp_tps": 3394.1, "tg_tokens": 128, "tg_s": 1.6888, "tg_tps": 75.2, "raw": {"pp": [0.16184503305703402, 0.1508498010225594, 0.15504808397963643], "tg": [1.7033594409003854, 1.8424865039996803, 2.1489964802749455], "tg1": [0.016280516982078552, 0.015342713799327612, 0.014542239718139172]}} {"tag": "gate+chunk", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-int4-g64", "dtype": "bfloat16", "compilation_config": "{\"inductor_compile_config\":{\"cpp.min_chunk_size\":65536}}", "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 80.04, "rss_mb": 8612.2, "rss_peak_mb": 8640.3, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.1866, "pp_tps": 2743.17, "tg_tokens": 128, "tg_s": 1.8041, "tg_tps": 70.4, "raw": {"pp": [0.2269656560383737, 0.19936348916962743, 0.18664512783288956], "tg": [1.8515523397363722, 1.8183003733865917, 1.8471249351277947], "tg1": [0.014217090792953968, 0.016116315964609385, 0.01467960188165307]}} {"tag": "gate+chunk+g128", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-int4-g128", "dtype": "bfloat16", "compilation_config": "{\"inductor_compile_config\":{\"cpp.min_chunk_size\":65536}}", "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 86.06, "rss_mb": 8056.8, "rss_peak_mb": 8084.4, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.1512, "pp_tps": 3387.19, "tg_tokens": 128, "tg_s": 1.6258, "tg_tps": 78.12, "raw": {"pp": [0.1577955186367035, 0.15690755285322666, 0.15115766506642103], "tg": [1.690211537759751, 1.6389126512221992, 1.6603844622150064], "tg1": [0.013693444896489382, 0.013548826798796654, 0.013116502668708563]}} {"tag": "w4a8+gate+chunk", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-gptq-g64", "dtype": "bfloat16", "compilation_config": "{\"inductor_compile_config\":{\"cpp.min_chunk_size\":65536}}", "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 79.15, "rss_mb": 9519.8, "rss_peak_mb": 9541.6, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.2452, "pp_tps": 2088.2, "tg_tokens": 128, "tg_s": 2.2477, "tg_tps": 56.5, "raw": {"pp": [0.24662079522386193, 0.2584368041716516, 0.24518772400915623], "tg": [2.265108836814761, 2.3531849440187216, 2.368000926449895], "tg1": [0.01957006100565195, 0.019077030941843987, 0.017433146946132183]}} {"tag": "base-repeat", "model": "/data/zhoutaichang/embedding_infer/models/spark2_5-int4-g64", "dtype": "bfloat16", "compilation_config": null, "detokenize": true, "quantization": null, "enforce_eager": false, "init_s": 51.8, "rss_mb": 8724.6, "rss_peak_mb": 8766.6, "threads_bind": "32-55", "omp_num_threads": "24", "pp_tokens": 512, "pp_s": 0.2006, "pp_tps": 2551.88, "tg_tokens": 128, "tg_s": 1.8666, "tg_tps": 68.04, "raw": {"pp": [0.23906681034713984, 0.2106484593823552, 0.20063663413748145], "tg": [1.9647751650772989, 1.9212167048826814, 1.8819844778627157], "tg1": [0.016026358120143414, 0.0173974996432662, 0.015383167192339897]}}