{ "tag": "bfloat16/None/eager=False", "wall_step_ms": 17.25, "tps": 57.97, "profiled_total_ms_per_step": 16.484, "buckets_ms_per_step": { "gemm": 10.661, "framework": 3.502, "other_op": 1.165, "attn": 1.157 }, "top": [ { "op": "aten::_weight_int4pack_mm_for_cpu", "bucket": "gemm", "ms_per_step": 8.8694, "calls_per_step": 113.0 }, { "op": "## Call CompiledFxGraph None ##", "bucket": "framework", "ms_per_step": 2.7736, "calls_per_step": 1.0 }, { "op": "aten::mm", "bucket": "gemm", "ms_per_step": 1.1547, "calls_per_step": 28.0 }, { "op": "vllm::cpu_int4_linear", "bucket": "gemm", "ms_per_step": 0.6367, "calls_per_step": 113.0 }, { "op": "vllm::unified_kv_cache_update", "bucket": "framework", "ms_per_step": 0.5477, "calls_per_step": 28.0 }, { "op": "vllm::unified_attention_with_output", "bucket": "attn", "ms_per_step": 0.5346, "calls_per_step": 28.0 }, { "op": "_C::cpu_attention_with_kv_cache", "bucket": "attn", "ms_per_step": 0.4428, "calls_per_step": 28.0 }, { "op": "aten::argmax", "bucket": "other_op", "ms_per_step": 0.2687, "calls_per_step": 1.0 }, { "op": "_C::cpu_attn_reshape_and_cache", "bucket": "attn", "ms_per_step": 0.1792, "calls_per_step": 28.0 }, { "op": "aten::slice", "bucket": "other_op", "ms_per_step": 0.1786, "calls_per_step": 230.0 }, { "op": "Pregraph bytecode", "bucket": "framework", "ms_per_step": 0.1727, "calls_per_step": 1.0 }, { "op": "aten::narrow", "bucket": "other_op", "ms_per_step": 0.1042, "calls_per_step": 112.0 }, { "op": "aten::as_strided", "bucket": "other_op", "ms_per_step": 0.0839, "calls_per_step": 234.0 }, { "op": "aten::copy_", "bucket": "other_op", "ms_per_step": 0.0833, "calls_per_step": 19.0 }, { "op": "aten::split", "bucket": "other_op", "ms_per_step": 0.0712, "calls_per_step": 56.0 }, { "op": "aten::chunk", "bucket": "other_op", "ms_per_step": 0.07, "calls_per_step": 56.0 }, { "op": "aten::view", "bucket": "other_op", "ms_per_step": 0.0631, "calls_per_step": 62.0 }, { "op": "aten::empty", "bucket": "other_op", "ms_per_step": 0.0551, "calls_per_step": 145.0 }, { "op": "aten::index", "bucket": "other_op", "ms_per_step": 0.0345, "calls_per_step": 2.0 }, { "op": "_C::compute_slot_mapping_kernel_impl", "bucket": "other_op", "ms_per_step": 0.022, "calls_per_step": 4.0 }, { "op": "aten::new_empty", "bucket": "other_op", "ms_per_step": 0.0187, "calls_per_step": 28.0 }, { "op": "_C::get_scheduler_metadata", "bucket": "other_op", "ms_per_step": 0.0184, "calls_per_step": 4.0 }, { "op": "aten::_to_copy", "bucket": "other_op", "ms_per_step": 0.0168, "calls_per_step": 5.0 }, { "op": "aten::add", "bucket": "other_op", "ms_per_step": 0.0115, "calls_per_step": 3.0 }, { "op": "aten::fill_", "bucket": "other_op", "ms_per_step": 0.0112, "calls_per_step": 12.0 }, { "op": "aten::resolve_conj", "bucket": "other_op", "ms_per_step": 0.0096, "calls_per_step": 59.0 }, { "op": "AOTDispatcher Runtime Wrapper Prologue", "bucket": "framework", "ms_per_step": 0.0077, "calls_per_step": 1.0 }, { "op": "aten::empty_strided", "bucket": "other_op", "ms_per_step": 0.0073, "calls_per_step": 5.0 }, { "op": "aten::to", "bucket": "other_op", "ms_per_step": 0.007, "calls_per_step": 8.0 }, { "op": "aten::sub", "bucket": "other_op", "ms_per_step": 0.0063, "calls_per_step": 1.0 }, { "op": "aten::lt", "bucket": "other_op", "ms_per_step": 0.0049, "calls_per_step": 1.0 }, { "op": "aten::index_select", "bucket": "other_op", "ms_per_step": 0.0043, "calls_per_step": 1.0 }, { "op": "aten::detach", "bucket": "other_op", "ms_per_step": 0.0037, "calls_per_step": 2.0 }, { "op": "aten::reshape", "bucket": "other_op", "ms_per_step": 0.0031, "calls_per_step": 4.0 }, { "op": "aten::alias", "bucket": "other_op", "ms_per_step": 0.0026, "calls_per_step": 1.0 } ] }