Files
vllm-omni/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/resource-summary.json
T

9841 lines
320 KiB
JSON

{
"telemetry": {
"spark-cpu-wsl": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-host\\telemetry.jsonl",
"samples": 2656,
"span_s": 3117.997078895569,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 2655,
"p50": 1.1173491477966309,
"p95": 1.4749903678894043,
"mean": 1.1743868470416454,
"min": 1.051551103591919,
"max": 6.049559831619263
},
"gaps_over_5s": 2,
"metrics": {
"host_available_bytes": {
"n": 2656,
"p50": 18036830208.0,
"p95": 22102822912.0,
"mean": 19110518421.590363,
"min": 16652177408.0,
"max": 26857451520.0
},
"host_used_bytes": {
"n": 2656,
"p50": 49733742592.0,
"p95": 50563702784.0,
"mean": 48662991210.40964,
"min": 40916058112.0,
"max": 51121332224.0
},
"host_cpu_percent": {
"n": 2656,
"p50": 64.9,
"p95": 76.0,
"mean": 65.03772590361446,
"min": 8.3,
"max": 96.1
},
"gpu_memory_mib": {
"n": 2655,
"p50": 4015.0,
"p95": 4135.0,
"mean": 3984.3966101694914,
"min": 3784.0,
"max": 4335.0
},
"gpu_temperature_c": {
"n": 2655,
"p50": 61.0,
"p95": 63.0,
"mean": 60.2241054613936,
"min": 50.0,
"max": 65.0
},
"gpu_power_w": {
"n": 2655,
"p50": 22.02,
"p95": 31.14,
"mean": 22.19570998116761,
"min": 12.25,
"max": 39.0
},
"gpu_sm_clock_mhz": {
"n": 2655,
"p50": 757.0,
"p95": 1687.0,
"mean": 825.1156308851224,
"min": 180.0,
"max": 2092.0
},
"gpu_mem_clock_mhz": {
"n": 2655,
"p50": 405.0,
"p95": 9001.0,
"mean": 928.2248587570622,
"min": 405.0,
"max": 9001.0
},
"gpu_utilization_percent": {
"n": 2655,
"p50": 6.0,
"p95": 23.0,
"mean": 7.565348399246704,
"min": 0.0,
"max": 85.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 68219.60971138597,
"energy_observed_duration_s": 3104.303554058075,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 14:56:03 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 634959"
]
}
}
},
"startup": {
"cache_condition": "existing disk/JIT cache; not cold disk",
"constructor_s": 20.766939496970735,
"host_launch_to_benchmark_report_start_s": 10.787997961044312,
"host_launch_to_first_request_s": 34.17721366882324,
"host_launch_to_first_output_s": 34.351301193237305,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "cad076712a854665-0",
"length_band": "short",
"concurrency": 1,
"wall_s": 8.226480221026577,
"ttft_s": 0.1740875244140625,
"total_wall_s": null,
"ttfa_ms": null,
"rtf_total": null
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=634959) INFO 09-22 14:04:42 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.14 s",
"INFO 09-22 14:04:42 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
},
"sustained": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-host\\telemetry.jsonl",
"samples": 1487,
"span_s": 1804.798624753952,
"requested_window": {
"start": 1790058336.0866268,
"stop": 1790060141.8543375
},
"sampling_gap_s": {
"n": 1486,
"p50": 1.1222476959228516,
"p95": 1.4839341640472412,
"mean": 1.2145347407496312,
"min": 1.0863194465637207,
"max": 6.049559831619263
},
"gaps_over_5s": 1,
"metrics": {
"host_available_bytes": {
"n": 1487,
"p50": 20670472192.0,
"p95": 22149316608.0,
"mean": 20109044567.284466,
"min": 16652177408.0,
"max": 22225444864.0
},
"host_used_bytes": {
"n": 1487,
"p50": 47103037440.0,
"p95": 50572922880.0,
"mean": 47664465064.71554,
"min": 45548064768.0,
"max": 51121332224.0
},
"host_cpu_percent": {
"n": 1487,
"p50": 64.0,
"p95": 75.1,
"mean": 64.75480833893745,
"min": 55.9,
"max": 88.9
},
"gpu_memory_mib": {
"n": 1486,
"p50": 3909.0,
"p95": 4060.0,
"mean": 3939.0666218034994,
"min": 3784.0,
"max": 4186.0
},
"gpu_temperature_c": {
"n": 1486,
"p50": 60.0,
"p95": 62.0,
"mean": 60.00403768506057,
"min": 57.0,
"max": 63.0
},
"gpu_power_w": {
"n": 1486,
"p50": 19.87,
"p95": 28.92,
"mean": 20.549764468371468,
"min": 13.62,
"max": 38.83
},
"gpu_sm_clock_mhz": {
"n": 1486,
"p50": 675.0,
"p95": 1702.0,
"mean": 798.8034993270525,
"min": 180.0,
"max": 2092.0
},
"gpu_mem_clock_mhz": {
"n": 1486,
"p50": 405.0,
"p95": 810.0,
"mean": 768.4259757738896,
"min": 405.0,
"max": 9001.0
},
"gpu_utilization_percent": {
"n": 1486,
"p50": 4.0,
"p95": 22.0,
"mean": 6.034320323014805,
"min": 0.0,
"max": 85.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 36497.229892824886,
"energy_observed_duration_s": 1797.0533385276794,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_first_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-host\\telemetry.jsonl",
"samples": 269,
"span_s": 299.3138289451599,
"requested_window": {
"start": 1790058336.0866268,
"stop": 1790058636.0866268
},
"sampling_gap_s": {
"n": 268,
"p50": 1.1171205043792725,
"p95": 1.1304068565368652,
"mean": 1.116842645317761,
"min": 1.086822748184204,
"max": 1.1373279094696045
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 269,
"p50": 17299808256.0,
"p95": 17939537920.0,
"mean": 17292257394.200745,
"min": 16652177408.0,
"max": 18291531776.0
},
"host_used_bytes": {
"n": 269,
"p50": 50473701376.0,
"p95": 50841006080.0,
"mean": 50481252237.799255,
"min": 49481977856.0,
"max": 51121332224.0
},
"host_cpu_percent": {
"n": 269,
"p50": 65.8,
"p95": 75.0,
"mean": 66.66617100371748,
"min": 60.6,
"max": 84.3
},
"gpu_memory_mib": {
"n": 269,
"p50": 3901.0,
"p95": 3993.0,
"mean": 3912.8624535315985,
"min": 3888.0,
"max": 4082.0
},
"gpu_temperature_c": {
"n": 269,
"p50": 62.0,
"p95": 62.0,
"mean": 61.84014869888476,
"min": 61.0,
"max": 62.0
},
"gpu_power_w": {
"n": 269,
"p50": 22.22,
"p95": 24.75,
"mean": 22.5328624535316,
"min": 20.92,
"max": 32.42
},
"gpu_sm_clock_mhz": {
"n": 269,
"p50": 690.0,
"p95": 847.0,
"mean": 709.6728624535316,
"min": 562.0,
"max": 1222.0
},
"gpu_mem_clock_mhz": {
"n": 269,
"p50": 405.0,
"p95": 405.0,
"mean": 406.5055762081784,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 269,
"p50": 7.0,
"p95": 16.0,
"mean": 8.345724907063197,
"min": 3.0,
"max": 52.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 6744.291670222283,
"energy_observed_duration_s": 299.3138289451599,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-host\\telemetry.jsonl",
"samples": 206,
"span_s": 299.2749743461609,
"requested_window": {
"start": 1790059841.8543375,
"stop": 1790060141.8543375
},
"sampling_gap_s": {
"n": 205,
"p50": 1.4662668704986572,
"p95": 1.5056633949279785,
"mean": 1.4598779236398092,
"min": 1.0879061222076416,
"max": 1.6294188499450684
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 206,
"p50": 22097182720.0,
"p95": 22198444032.0,
"mean": 22097246526.13592,
"min": 21917319168.0,
"max": 22225444864.0
},
"host_used_bytes": {
"n": 206,
"p50": 45675880448.0,
"p95": 45806276608.0,
"mean": 45676263105.864075,
"min": 45548064768.0,
"max": 45856190464.0
},
"host_cpu_percent": {
"n": 206,
"p50": 59.1,
"p95": 64.6,
"mean": 59.76699029126213,
"min": 56.0,
"max": 69.3
},
"gpu_memory_mib": {
"n": 206,
"p50": 4051.0,
"p95": 4064.0,
"mean": 4053.7281553398057,
"min": 4051.0,
"max": 4070.0
},
"gpu_temperature_c": {
"n": 206,
"p50": 58.0,
"p95": 58.0,
"mean": 57.88834951456311,
"min": 57.0,
"max": 58.0
},
"gpu_power_w": {
"n": 206,
"p50": 17.33,
"p95": 20.17,
"mean": 17.169466019417474,
"min": 13.62,
"max": 20.65
},
"gpu_sm_clock_mhz": {
"n": 206,
"p50": 1200.0,
"p95": 1725.0,
"mean": 1017.8009708737864,
"min": 180.0,
"max": 1740.0
},
"gpu_mem_clock_mhz": {
"n": 206,
"p50": 810.0,
"p95": 810.0,
"mean": 615.3640776699029,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 206,
"p50": 0.0,
"p95": 2.0,
"mean": 0.5679611650485437,
"min": 0.0,
"max": 3.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 5140.545204623938,
"energy_observed_duration_s": 299.2749743461609,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_60s": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-host\\telemetry.jsonl",
"samples": 42,
"span_s": 59.30613565444946,
"requested_window": {
"start": 1790060081.8543375,
"stop": 1790060141.8543375
},
"sampling_gap_s": {
"n": 41,
"p50": 1.469386339187622,
"p95": 1.5250725746154785,
"mean": 1.4464911135231577,
"min": 1.1175227165222168,
"max": 1.6294188499450684
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 42,
"p50": 22063603712.0,
"p95": 22080110592.0,
"mean": 22060072082.285713,
"min": 22016536576.0,
"max": 22081986560.0
},
"host_used_bytes": {
"n": 42,
"p50": 45709516800.0,
"p95": 45743243264.0,
"mean": 45713437549.71429,
"min": 45691523072.0,
"max": 45756973056.0
},
"host_cpu_percent": {
"n": 42,
"p50": 59.4,
"p95": 63.5,
"mean": 59.75714285714286,
"min": 56.8,
"max": 65.5
},
"gpu_memory_mib": {
"n": 42,
"p50": 4051.0,
"p95": 4064.0,
"mean": 4053.404761904762,
"min": 4051.0,
"max": 4064.0
},
"gpu_temperature_c": {
"n": 42,
"p50": 58.0,
"p95": 58.0,
"mean": 57.785714285714285,
"min": 57.0,
"max": 58.0
},
"gpu_power_w": {
"n": 42,
"p50": 16.59,
"p95": 20.24,
"mean": 16.836904761904762,
"min": 13.68,
"max": 20.3
},
"gpu_sm_clock_mhz": {
"n": 42,
"p50": 757.0,
"p95": 1725.0,
"mean": 962.5952380952381,
"min": 180.0,
"max": 1732.0
},
"gpu_mem_clock_mhz": {
"n": 42,
"p50": 405.0,
"p95": 810.0,
"mean": 588.2142857142857,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 42,
"p50": 0.0,
"p95": 2.0,
"mean": 0.6190476190476191,
"min": 0.0,
"max": 2.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 1001.230226817131,
"energy_observed_duration_s": 59.30613565444946,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_request_summary": {
"n": 202,
"rtf_above_one_requests": 0,
"simulated_playback_deficit_requests": 0,
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"scope": "All sustained requests; timing/real-time gates are separate from successful completion."
},
"sustained_request_trends": {
"first_5min": {
"n": 33,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 33,
"p50": 9.256840700982139,
"p95": 9.541978118999396,
"mean": 9.268286953330971,
"min": 8.972634237026796,
"max": 9.617705173033755
},
"ttft_s": {
"n": 33,
"p50": 0.7878987789154053,
"p95": 0.8170230388641357,
"mean": 0.7897633278008663,
"min": 0.7565879821777344,
"max": 0.8395428657531738
},
"decode_tok_per_s": {
"n": 33,
"p50": 14.99183990995157,
"p95": 15.469756966909298,
"mean": 14.98472165328966,
"min": 14.38821297813298,
"max": 15.53023769770836
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_5min": {
"n": 35,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 35,
"p50": 8.38500963698607,
"p95": 8.551528260984924,
"mean": 8.401563459719597,
"min": 8.250801810994744,
"max": 8.581972534011584
},
"ttft_s": {
"n": 35,
"p50": 0.7843046188354492,
"p95": 0.8108699321746826,
"mean": 0.7838305132729667,
"min": 0.7457151412963867,
"max": 0.8254172801971436
},
"decode_tok_per_s": {
"n": 35,
"p50": 16.723442363642626,
"p95": 16.889158941288905,
"mean": 16.673895675463257,
"min": 16.371648751212792,
"max": 16.9782480827634
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_60s": {
"n": 7,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 7,
"p50": 8.373090737033635,
"p95": 8.49373803200433,
"mean": 8.398709651872716,
"min": 8.307218600006308,
"max": 8.49373803200433
},
"ttft_s": {
"n": 7,
"p50": 0.7820346355438232,
"p95": 0.8010997772216797,
"mean": 0.7800593716757638,
"min": 0.7632486820220947,
"max": 0.8010997772216797
},
"decode_tok_per_s": {
"n": 7,
"p50": 16.734281893896224,
"p95": 16.882191326698376,
"mean": 16.671197765155018,
"min": 16.493817921742796,
"max": 16.882191326698376
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
}
}
},
"spark-cpu-wsl-reliability": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-reliability-host\\telemetry.jsonl",
"samples": 78,
"span_s": 84.00985193252563,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 77,
"p50": 1.0758404731750488,
"p95": 1.1334936618804932,
"mean": 1.0910370380847485,
"min": 1.0588123798370361,
"max": 1.1430747509002686
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 78,
"p50": 19191267328.0,
"p95": 27231408128.0,
"mean": 22243715072.0,
"min": 18700869632.0,
"max": 27919523840.0
},
"host_used_bytes": {
"n": 78,
"p50": 48436744192.0,
"p95": 48912994304.0,
"mean": 45529794560.0,
"min": 39853985792.0,
"max": 49072640000.0
},
"host_cpu_percent": {
"n": 78,
"p50": 29.1,
"p95": 81.1,
"mean": 44.937179487179485,
"min": 15.8,
"max": 83.3
},
"gpu_memory_mib": {
"n": 78,
"p50": 2057.0,
"p95": 2083.0,
"mean": 2058.74358974359,
"min": 2043.0,
"max": 2085.0
},
"gpu_temperature_c": {
"n": 78,
"p50": 62.0,
"p95": 65.0,
"mean": 62.65384615384615,
"min": 61.0,
"max": 65.0
},
"gpu_power_w": {
"n": 78,
"p50": 33.98,
"p95": 36.76,
"mean": 33.91833333333334,
"min": 20.45,
"max": 43.79
},
"gpu_sm_clock_mhz": {
"n": 78,
"p50": 1020.0,
"p95": 1995.0,
"mean": 1119.179487179487,
"min": 862.0,
"max": 2010.0
},
"gpu_mem_clock_mhz": {
"n": 78,
"p50": 9001.0,
"p95": 9001.0,
"mean": 9001.0,
"min": 9001.0,
"max": 9001.0
},
"gpu_utilization_percent": {
"n": 78,
"p50": 18.0,
"p95": 20.0,
"mean": 17.935897435897434,
"min": 13.0,
"max": 21.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 2853.088289884329,
"energy_observed_duration_s": 84.00985193252563,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": null,
"host_launch_to_benchmark_report_start_s": 14.845646142959595,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1329965) INFO 09-22 17:27:01 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.33 s",
"INFO 09-22 17:27:03 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cpu-wsl-trace-full": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-trace-full-host\\telemetry.jsonl",
"samples": 261,
"span_s": 373.485059261322,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 260,
"p50": 1.4629204273223877,
"p95": 1.4849483966827393,
"mean": 1.436480997158931,
"min": 1.055802822113037,
"max": 1.5934369564056396
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 261,
"p50": 5460541440.0,
"p95": 25058160640.0,
"mean": 8166366196.229885,
"min": 493735936.0,
"max": 26082906112.0
},
"host_used_bytes": {
"n": 261,
"p50": 62312968192.0,
"p95": 67189768192.0,
"mean": 59607143435.77012,
"min": 41690603520.0,
"max": 67279773696.0
},
"host_cpu_percent": {
"n": 261,
"p50": 11.4,
"p95": 60.0,
"mean": 21.035632183908046,
"min": 6.2,
"max": 68.7
},
"gpu_memory_mib": {
"n": 261,
"p50": 4051.0,
"p95": 4064.0,
"mean": 3967.448275862069,
"min": 3754.0,
"max": 4083.0
},
"gpu_temperature_c": {
"n": 261,
"p50": 51.0,
"p95": 58.0,
"mean": 51.758620689655174,
"min": 49.0,
"max": 58.0
},
"gpu_power_w": {
"n": 261,
"p50": 17.11,
"p95": 19.9,
"mean": 16.56942528735632,
"min": 10.8,
"max": 27.93
},
"gpu_sm_clock_mhz": {
"n": 261,
"p50": 1597.0,
"p95": 1725.0,
"mean": 1191.9540229885058,
"min": 180.0,
"max": 1755.0
},
"gpu_mem_clock_mhz": {
"n": 261,
"p50": 810.0,
"p95": 810.0,
"mean": 671.8965517241379,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 261,
"p50": 0.0,
"p95": 3.0,
"mean": 0.5938697318007663,
"min": 0.0,
"max": 5.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 6184.594364783764,
"energy_observed_duration_s": 373.485059261322,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 15:02:42 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 642362"
]
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 21.080843324016314,
"host_launch_to_benchmark_report_start_s": 10.202934503555298,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=642362) INFO 09-22 14:56:59 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.19 s",
"INFO 09-22 14:57:01 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cpu-wsl-trace": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-trace-host\\telemetry.jsonl",
"samples": 75,
"span_s": 79.53476667404175,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 74,
"p50": 1.063019037246704,
"p95": 1.1171836853027344,
"mean": 1.0747941442438074,
"min": 1.0534603595733643,
"max": 1.126281976699829
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 75,
"p50": 17925554176.0,
"p95": 26398646272.0,
"mean": 19642509230.08,
"min": 15018422272.0,
"max": 26994212864.0
},
"host_used_bytes": {
"n": 75,
"p50": 49847955456.0,
"p95": 52735946752.0,
"mean": 48131000401.92,
"min": 40779296768.0,
"max": 52755087360.0
},
"host_cpu_percent": {
"n": 75,
"p50": 13.9,
"p95": 61.8,
"mean": 24.329333333333334,
"min": 6.9,
"max": 65.7
},
"gpu_memory_mib": {
"n": 75,
"p50": 1376.0,
"p95": 1390.0,
"mean": 1378.7333333333333,
"min": 1371.0,
"max": 1390.0
},
"gpu_temperature_c": {
"n": 75,
"p50": 56.0,
"p95": 57.0,
"mean": 55.29333333333334,
"min": 53.0,
"max": 57.0
},
"gpu_power_w": {
"n": 75,
"p50": 13.85,
"p95": 15.37,
"mean": 14.042000000000002,
"min": 12.39,
"max": 23.26
},
"gpu_sm_clock_mhz": {
"n": 75,
"p50": 375.0,
"p95": 442.0,
"mean": 380.05333333333334,
"min": 345.0,
"max": 465.0
},
"gpu_mem_clock_mhz": {
"n": 75,
"p50": 405.0,
"p95": 405.0,
"mean": 405.0,
"min": 405.0,
"max": 405.0
},
"gpu_utilization_percent": {
"n": 75,
"p50": 1.0,
"p95": 2.0,
"mean": 1.24,
"min": 1.0,
"max": 6.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 1116.2872086429595,
"energy_observed_duration_s": 79.53476667404175,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 18:39:18 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 1547774"
]
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 20.63935291103553,
"host_launch_to_benchmark_report_start_s": 10.191941022872925,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1547774) INFO 09-22 18:38:28 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.15 s",
"INFO 09-22 18:38:30 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cuda-windows": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-host\\telemetry.jsonl",
"samples": 2201,
"span_s": 2347.7642068862915,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 2200,
"p50": 1.0636231899261475,
"p95": 1.089616060256958,
"mean": 1.067165548584678,
"min": 1.0493698120117188,
"max": 1.1246931552886963
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 2201,
"p50": 19926433792.0,
"p95": 21706616832.0,
"mean": 19998352203.020447,
"min": 17776742400.0,
"max": 27779530752.0
},
"host_used_bytes": {
"n": 2201,
"p50": 47847075840.0,
"p95": 48937619456.0,
"mean": 47775157428.97955,
"min": 39993978880.0,
"max": 49996767232.0
},
"host_cpu_percent": {
"n": 2201,
"p50": 18.3,
"p95": 29.3,
"mean": 19.39981826442526,
"min": 9.6,
"max": 59.1
},
"gpu_memory_mib": {
"n": 2201,
"p50": 13849.0,
"p95": 14362.0,
"mean": 13265.120854157201,
"min": 2505.0,
"max": 14792.0
},
"gpu_temperature_c": {
"n": 2201,
"p50": 69.0,
"p95": 72.0,
"mean": 68.55792821444798,
"min": 54.0,
"max": 73.0
},
"gpu_power_w": {
"n": 2201,
"p50": 92.61,
"p95": 99.9,
"mean": 87.05158109950023,
"min": 15.41,
"max": 156.07
},
"gpu_sm_clock_mhz": {
"n": 2201,
"p50": 1545.0,
"p95": 2565.0,
"mean": 1662.311676510677,
"min": 367.0,
"max": 2670.0
},
"gpu_mem_clock_mhz": {
"n": 2201,
"p50": 14001.0,
"p95": 14001.0,
"mean": 12002.372103589278,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 2201,
"p50": 91.0,
"p95": 96.0,
"mean": 68.83371194911403,
"min": 1.0,
"max": 99.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 204471.93893461226,
"energy_observed_duration_s": 2347.7642068862915,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "existing disk/JIT cache; not cold disk",
"constructor_s": 95.1554521000362,
"host_launch_to_benchmark_report_start_s": 19.451645135879517,
"host_launch_to_first_request_s": 175.59016942977905,
"host_launch_to_first_output_s": 175.88464784622192,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "7d4c38429f554f02-0",
"length_band": "short",
"concurrency": 1,
"wall_s": 2.8171001000446267,
"ttft_s": 0.2944784164428711,
"total_wall_s": null,
"ttfa_ms": null,
"rtf_total": null
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=16532) INFO 09-22 16:34:42 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 53.142006 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=16532) INFO 09-22 16:34:44 [core.py:368] init engine (profile, create kv cache, warmup model) took 1.70 s",
"INFO 09-22 16:34:46 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
},
"sustained": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-host\\telemetry.jsonl",
"samples": 1688,
"span_s": 1802.594036579132,
"requested_window": {
"start": 1790066445.1752505,
"stop": 1790068248.4246356
},
"sampling_gap_s": {
"n": 1687,
"p50": 1.0646803379058838,
"p95": 1.0907464027404785,
"mean": 1.0685204721868002,
"min": 1.0493698120117188,
"max": 1.1246931552886963
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 1688,
"p50": 19596402688.0,
"p95": 20313821184.0,
"mean": 19640543234.42654,
"min": 17776742400.0,
"max": 20405301248.0
},
"host_used_bytes": {
"n": 1688,
"p50": 48177037312.0,
"p95": 48960778240.0,
"mean": 48132966397.57346,
"min": 47368208384.0,
"max": 49996767232.0
},
"host_cpu_percent": {
"n": 1688,
"p50": 18.4,
"p95": 30.0,
"mean": 19.583116113744076,
"min": 11.6,
"max": 59.1
},
"gpu_memory_mib": {
"n": 1688,
"p50": 13881.0,
"p95": 14364.0,
"mean": 13923.961492890996,
"min": 13740.0,
"max": 14792.0
},
"gpu_temperature_c": {
"n": 1688,
"p50": 69.0,
"p95": 73.0,
"mean": 69.60011848341232,
"min": 66.0,
"max": 73.0
},
"gpu_power_w": {
"n": 1688,
"p50": 92.86,
"p95": 99.84,
"mean": 92.00824644549763,
"min": 30.09,
"max": 156.07
},
"gpu_sm_clock_mhz": {
"n": 1688,
"p50": 1612.0,
"p95": 2572.0,
"mean": 1764.8382701421801,
"min": 577.0,
"max": 2670.0
},
"gpu_mem_clock_mhz": {
"n": 1688,
"p50": 14001.0,
"p95": 14001.0,
"mean": 12689.779028436018,
"min": 9001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 1688,
"p50": 92.0,
"p95": 96.0,
"mean": 71.44135071090048,
"min": 4.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 165841.9560630107,
"energy_observed_duration_s": 1802.594036579132,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_first_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-host\\telemetry.jsonl",
"samples": 283,
"span_s": 299.1382167339325,
"requested_window": {
"start": 1790066445.1752505,
"stop": 1790066745.1752505
},
"sampling_gap_s": {
"n": 282,
"p50": 1.0596892833709717,
"p95": 1.0724682807922363,
"mean": 1.0607738182054345,
"min": 1.0499680042266846,
"max": 1.0960776805877686
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 283,
"p50": 20052283392.0,
"p95": 20349710336.0,
"mean": 20110681308.720848,
"min": 19862536192.0,
"max": 20397076480.0
},
"host_used_bytes": {
"n": 283,
"p50": 47721226240.0,
"p95": 47869759488.0,
"mean": 47662828323.27915,
"min": 47376433152.0,
"max": 47910973440.0
},
"host_cpu_percent": {
"n": 283,
"p50": 16.3,
"p95": 24.7,
"mean": 17.168904593639578,
"min": 11.8,
"max": 32.9
},
"gpu_memory_mib": {
"n": 283,
"p50": 13831.0,
"p95": 13994.0,
"mean": 13835.088339222615,
"min": 13750.0,
"max": 14143.0
},
"gpu_temperature_c": {
"n": 283,
"p50": 68.0,
"p95": 68.0,
"mean": 67.73851590106007,
"min": 66.0,
"max": 69.0
},
"gpu_power_w": {
"n": 283,
"p50": 94.11,
"p95": 100.2,
"mean": 93.25455830388692,
"min": 66.75,
"max": 156.07
},
"gpu_sm_clock_mhz": {
"n": 283,
"p50": 1387.0,
"p95": 1620.0,
"mean": 1333.9187279151943,
"min": 592.0,
"max": 1725.0
},
"gpu_mem_clock_mhz": {
"n": 283,
"p50": 14001.0,
"p95": 14001.0,
"mean": 13405.265017667845,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 283,
"p50": 94.0,
"p95": 96.0,
"mean": 93.4452296819788,
"min": 32.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 27894.00177993536,
"energy_observed_duration_s": 299.1382167339325,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-host\\telemetry.jsonl",
"samples": 281,
"span_s": 298.9066114425659,
"requested_window": {
"start": 1790067948.4246356,
"stop": 1790068248.4246356
},
"sampling_gap_s": {
"n": 280,
"p50": 1.0666837692260742,
"p95": 1.0804500579833984,
"mean": 1.0675236122948784,
"min": 1.051903247833252,
"max": 1.1027801036834717
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 19385311232.0,
"p95": 20323061760.0,
"mean": 19537713275.900356,
"min": 18818297856.0,
"max": 20405301248.0
},
"host_used_bytes": {
"n": 281,
"p50": 48388198400.0,
"p95": 48881623040.0,
"mean": 48235796356.09965,
"min": 47368208384.0,
"max": 48955211776.0
},
"host_cpu_percent": {
"n": 281,
"p50": 21.5,
"p95": 32.2,
"mean": 22.656939501779362,
"min": 12.3,
"max": 51.8
},
"gpu_memory_mib": {
"n": 281,
"p50": 14359.0,
"p95": 14420.0,
"mean": 14205.832740213524,
"min": 13876.0,
"max": 14792.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 69.0,
"p95": 71.0,
"mean": 69.33096085409252,
"min": 67.0,
"max": 72.0
},
"gpu_power_w": {
"n": 281,
"p50": 88.92,
"p95": 96.59,
"mean": 88.63871886120997,
"min": 62.69,
"max": 109.28
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1755.0,
"p95": 2220.0,
"mean": 1716.017793594306,
"min": 705.0,
"max": 2542.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 14001.0,
"p95": 14001.0,
"mean": 13480.266903914591,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 70.0,
"p95": 95.0,
"mean": 72.45907473309609,
"min": 29.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 26489.09294901371,
"energy_observed_duration_s": 298.9066114425659,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_60s": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-host\\telemetry.jsonl",
"samples": 56,
"span_s": 58.6842257976532,
"requested_window": {
"start": 1790068188.4246356,
"stop": 1790068248.4246356
},
"sampling_gap_s": {
"n": 55,
"p50": 1.06699800491333,
"p95": 1.076951026916504,
"mean": 1.0669859235936945,
"min": 1.0576961040496826,
"max": 1.0782222747802734
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 56,
"p50": 19369029632.0,
"p95": 19516518400.0,
"mean": 19378764288.0,
"min": 19283746816.0,
"max": 19525857280.0
},
"host_used_bytes": {
"n": 56,
"p50": 48393977856.0,
"p95": 48487014400.0,
"mean": 48394745344.0,
"min": 48247652352.0,
"max": 48489762816.0
},
"host_cpu_percent": {
"n": 56,
"p50": 23.0,
"p95": 34.8,
"mean": 23.707142857142856,
"min": 17.3,
"max": 36.4
},
"gpu_memory_mib": {
"n": 56,
"p50": 14364.0,
"p95": 14398.0,
"mean": 14372.05357142857,
"min": 14360.0,
"max": 14403.0
},
"gpu_temperature_c": {
"n": 56,
"p50": 69.0,
"p95": 70.0,
"mean": 69.03571428571429,
"min": 68.0,
"max": 70.0
},
"gpu_power_w": {
"n": 56,
"p50": 88.29,
"p95": 93.21,
"mean": 87.79767857142858,
"min": 62.69,
"max": 97.77
},
"gpu_sm_clock_mhz": {
"n": 56,
"p50": 1882.0,
"p95": 2160.0,
"mean": 1825.4464285714287,
"min": 1290.0,
"max": 2220.0
},
"gpu_mem_clock_mhz": {
"n": 56,
"p50": 14001.0,
"p95": 14001.0,
"mean": 14001.0,
"min": 14001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 56,
"p50": 67.0,
"p95": 96.0,
"mean": 69.08928571428571,
"min": 49.0,
"max": 96.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 5152.711587886811,
"energy_observed_duration_s": 58.6842257976532,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_request_summary": {
"n": 505,
"rtf_above_one_requests": 0,
"simulated_playback_deficit_requests": 0,
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"scope": "All sustained requests; timing/real-time gates are separate from successful completion."
},
"sustained_request_trends": {
"first_5min": {
"n": 108,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 108,
"p50": 2.7807982999947853,
"p95": 2.898030300042592,
"mean": 2.786717650926827,
"min": 2.655930700013414,
"max": 2.9437619999516755
},
"ttft_s": {
"n": 108,
"p50": 0.10055947303771973,
"p95": 0.15271973609924316,
"mean": 0.10645115596276743,
"min": 0.0787513256072998,
"max": 0.16637706756591797
},
"decode_tok_per_s": {
"n": 108,
"p50": 47.49805750540161,
"p95": 48.76951214063522,
"mean": 47.408907407180735,
"min": 44.712782088181136,
"max": 50.797316197092705
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_5min": {
"n": 81,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 81,
"p50": 3.7326299999840558,
"p95": 4.998527799965814,
"mean": 3.690130462964859,
"min": 2.685690899961628,
"max": 5.357077699969523
},
"ttft_s": {
"n": 81,
"p50": 0.10200071334838867,
"p95": 0.20329570770263672,
"mean": 0.11879588056493688,
"min": 0.08286523818969727,
"max": 0.22017502784729004
},
"decode_tok_per_s": {
"n": 81,
"p50": 34.814652233277954,
"p95": 47.246609572067236,
"mean": 37.01973046456721,
"min": 24.33034523293389,
"max": 49.70298739548921
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_60s": {
"n": 14,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 14,
"p50": 4.232541100005619,
"p95": 4.5487276000203565,
"mean": 4.183153571432091,
"min": 3.6028479999513365,
"max": 4.5487276000203565
},
"ttft_s": {
"n": 14,
"p50": 0.11452126502990723,
"p95": 0.2080843448638916,
"mean": 0.14110531125749862,
"min": 0.09841012954711914,
"max": 0.2080843448638916
},
"decode_tok_per_s": {
"n": 14,
"p50": 30.737668817917104,
"p95": 37.36023107549658,
"mean": 31.568899842729078,
"min": 29.22236021818931,
"max": 37.36023107549658
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
}
}
},
"spark-cuda-windows-reliability": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-reliability-host\\telemetry.jsonl",
"samples": 137,
"span_s": 145.31322288513184,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 136,
"p50": 1.06695556640625,
"p95": 1.083505392074585,
"mean": 1.068479580037734,
"min": 1.054593801498413,
"max": 1.1115479469299316
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 137,
"p50": 19596165120.0,
"p95": 21113839616.0,
"mean": 19645765826.335766,
"min": 17700904960.0,
"max": 21161598976.0
},
"host_used_bytes": {
"n": 137,
"p50": 48177344512.0,
"p95": 49858494464.0,
"mean": 48127743805.66423,
"min": 46611910656.0,
"max": 50072604672.0
},
"host_cpu_percent": {
"n": 137,
"p50": 18.1,
"p95": 30.5,
"mean": 19.57810218978102,
"min": 8.2,
"max": 52.1
},
"gpu_memory_mib": {
"n": 137,
"p50": 11063.0,
"p95": 13391.0,
"mean": 7452.510948905109,
"min": 2059.0,
"max": 13399.0
},
"gpu_temperature_c": {
"n": 137,
"p50": 56.0,
"p95": 61.0,
"mean": 56.78102189781022,
"min": 54.0,
"max": 61.0
},
"gpu_power_w": {
"n": 137,
"p50": 22.25,
"p95": 85.0,
"mean": 28.36722627737226,
"min": 13.66,
"max": 90.72
},
"gpu_sm_clock_mhz": {
"n": 137,
"p50": 690.0,
"p95": 1995.0,
"mean": 856.0583941605839,
"min": 345.0,
"max": 2340.0
},
"gpu_mem_clock_mhz": {
"n": 137,
"p50": 810.0,
"p95": 14001.0,
"mean": 4244.204379562044,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 137,
"p50": 12.0,
"p95": 51.0,
"mean": 17.583941605839417,
"min": 0.0,
"max": 63.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 4123.522332173586,
"energy_observed_duration_s": 145.31322288513184,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": null,
"host_launch_to_benchmark_report_start_s": 22.86827850341797,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=18880) INFO 09-22 17:31:18 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 58.143696 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=18880) INFO 09-22 17:31:21 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.61 s",
"INFO 09-22 17:31:23 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cuda-windows-trace": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-trace-host\\telemetry.jsonl",
"samples": 246,
"span_s": 261.54950618743896,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 245,
"p50": 1.0664939880371094,
"p95": 1.079019546508789,
"mean": 1.0675490048466896,
"min": 1.0556397438049316,
"max": 1.0874676704406738
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 246,
"p50": 15509241856.0,
"p95": 17894076416.0,
"mean": 15471848489.626017,
"min": 12033744896.0,
"max": 18053410816.0
},
"host_used_bytes": {
"n": 246,
"p50": 52228902912.0,
"p95": 55494774784.0,
"mean": 52301661142.373985,
"min": 49720098816.0,
"max": 55739764736.0
},
"host_cpu_percent": {
"n": 246,
"p50": 21.0,
"p95": 30.4,
"mean": 21.958536585365856,
"min": 14.5,
"max": 41.7
},
"gpu_memory_mib": {
"n": 246,
"p50": 12011.0,
"p95": 14392.0,
"mean": 9543.382113821139,
"min": 3209.0,
"max": 14410.0
},
"gpu_temperature_c": {
"n": 246,
"p50": 60.0,
"p95": 63.0,
"mean": 60.21951219512195,
"min": 58.0,
"max": 66.0
},
"gpu_power_w": {
"n": 246,
"p50": 34.08,
"p95": 83.26,
"mean": 37.171341463414635,
"min": 21.14,
"max": 102.64
},
"gpu_sm_clock_mhz": {
"n": 246,
"p50": 1087.0,
"p95": 2002.0,
"mean": 1391.4878048780488,
"min": 885.0,
"max": 2527.0
},
"gpu_mem_clock_mhz": {
"n": 246,
"p50": 9001.0,
"p95": 10820.0,
"mean": 9252.296747967479,
"min": 9001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 246,
"p50": 19.0,
"p95": 42.0,
"mean": 21.13821138211382,
"min": 16.0,
"max": 69.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 9728.738581544161,
"energy_observed_duration_s": 261.54950618743896,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 105.25394540000707,
"host_launch_to_benchmark_report_start_s": 61.28051710128784,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=10972) INFO 09-22 17:15:47 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 66.054303 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=10972) INFO 09-22 17:15:51 [core.py:368] init engine (profile, create kv cache, warmup model) took 3.83 s",
"INFO 09-22 17:15:53 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cuda-wsl": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-host\\telemetry.jsonl",
"samples": 2056,
"span_s": 2204.0161941051483,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 2055,
"p50": 1.065972089767456,
"p95": 1.0831704139709473,
"mean": 1.0725139630681988,
"min": 1.05218505859375,
"max": 4.844167709350586
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 2056,
"p50": 14104461312.0,
"p95": 15808049152.0,
"mean": 14372428616.715954,
"min": 12638658560.0,
"max": 23690711040.0
},
"host_used_bytes": {
"n": 2056,
"p50": 53668892672.0,
"p95": 54310875136.0,
"mean": 53401081015.28405,
"min": 44082798592.0,
"max": 55134851072.0
},
"host_cpu_percent": {
"n": 2056,
"p50": 19.5,
"p95": 31.1,
"mean": 20.78696498054475,
"min": 10.6,
"max": 55.1
},
"gpu_memory_mib": {
"n": 2056,
"p50": 14870.0,
"p95": 15105.0,
"mean": 14705.909046692606,
"min": 3648.0,
"max": 15368.0
},
"gpu_temperature_c": {
"n": 2056,
"p50": 68.0,
"p95": 71.0,
"mean": 68.24854085603113,
"min": 53.0,
"max": 72.0
},
"gpu_power_w": {
"n": 2056,
"p50": 88.6,
"p95": 94.31,
"mean": 87.01821498054476,
"min": 11.69,
"max": 109.39
},
"gpu_sm_clock_mhz": {
"n": 2056,
"p50": 1432.0,
"p95": 1582.0,
"mean": 1354.3278210116732,
"min": 180.0,
"max": 2047.0
},
"gpu_mem_clock_mhz": {
"n": 2056,
"p50": 10820.0,
"p95": 14001.0,
"mean": 11790.905642023346,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 2056,
"p50": 95.0,
"p95": 96.0,
"mean": 92.63229571984436,
"min": 0.0,
"max": 99.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 191165.87706233264,
"energy_observed_duration_s": 2204.0161941051483,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 15:39:49 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 643743"
]
}
}
},
"startup": {
"cache_condition": "existing disk/JIT cache; not cold disk",
"constructor_s": 23.692915348976385,
"host_launch_to_benchmark_report_start_s": 15.168210983276367,
"host_launch_to_first_request_s": 46.67540693283081,
"host_launch_to_first_output_s": 46.92364978790283,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "ce6cda4496404e5e-0",
"length_band": "short",
"concurrency": 1,
"wall_s": 2.710444374999497,
"ttft_s": 0.24824285507202148,
"total_wall_s": null,
"ttfa_ms": null,
"rtf_total": null
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=643743) INFO 09-22 15:03:48 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 1.947207 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=643743) INFO 09-22 15:03:53 [core.py:368] init engine (profile, create kv cache, warmup model) took 4.52 s",
"INFO 09-22 15:03:54 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
},
"sustained": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-host\\telemetry.jsonl",
"samples": 1689,
"span_s": 1800.6791484355927,
"requested_window": {
"start": 1790060974.1581197,
"stop": 1790062776.7599235
},
"sampling_gap_s": {
"n": 1688,
"p50": 1.0649433135986328,
"p95": 1.0802435874938965,
"mean": 1.0667530500210858,
"min": 1.05218505859375,
"max": 1.1284832954406738
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 1689,
"p50": 14110593024.0,
"p95": 15674961920.0,
"mean": 14291342197.162819,
"min": 13249519616.0,
"max": 15812558848.0
},
"host_used_bytes": {
"n": 1689,
"p50": 53662916608.0,
"p95": 54100258816.0,
"mean": 53482167434.83718,
"min": 51960950784.0,
"max": 54523990016.0
},
"host_cpu_percent": {
"n": 1689,
"p50": 19.3,
"p95": 29.0,
"mean": 20.175429248075787,
"min": 12.7,
"max": 46.1
},
"gpu_memory_mib": {
"n": 1689,
"p50": 14857.0,
"p95": 15023.0,
"mean": 14850.264061574897,
"min": 14675.0,
"max": 15278.0
},
"gpu_temperature_c": {
"n": 1689,
"p50": 68.0,
"p95": 70.0,
"mean": 68.38365896980461,
"min": 66.0,
"max": 72.0
},
"gpu_power_w": {
"n": 1689,
"p50": 88.53,
"p95": 93.74,
"mean": 88.1987981053878,
"min": 61.29,
"max": 104.67
},
"gpu_sm_clock_mhz": {
"n": 1689,
"p50": 1425.0,
"p95": 1560.0,
"mean": 1348.1030195381882,
"min": 697.0,
"max": 2047.0
},
"gpu_mem_clock_mhz": {
"n": 1689,
"p50": 10820.0,
"p95": 14001.0,
"mean": 11891.633510953227,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 1689,
"p50": 95.0,
"p95": 96.0,
"mean": 94.71936056838366,
"min": 67.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 158811.21432546497,
"energy_observed_duration_s": 1800.6791484355927,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_first_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-host\\telemetry.jsonl",
"samples": 281,
"span_s": 298.2404053211212,
"requested_window": {
"start": 1790060974.1581197,
"stop": 1790061274.1581197
},
"sampling_gap_s": {
"n": 280,
"p50": 1.0635502338409424,
"p95": 1.0772101879119873,
"mean": 1.06514430471829,
"min": 1.05218505859375,
"max": 1.0908541679382324
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 14036983808.0,
"p95": 14184722432.0,
"mean": 13891328721.537367,
"min": 13249519616.0,
"max": 14279856128.0
},
"host_used_bytes": {
"n": 281,
"p50": 53736525824.0,
"p95": 54413148160.0,
"mean": 53882180910.46263,
"min": 53493653504.0,
"max": 54523990016.0
},
"host_cpu_percent": {
"n": 281,
"p50": 17.9,
"p95": 25.9,
"mean": 18.85266903914591,
"min": 13.2,
"max": 42.7
},
"gpu_memory_mib": {
"n": 281,
"p50": 14943.0,
"p95": 15086.0,
"mean": 14974.142348754449,
"min": 14909.0,
"max": 15278.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 68.0,
"p95": 70.0,
"mean": 68.44839857651246,
"min": 67.0,
"max": 72.0
},
"gpu_power_w": {
"n": 281,
"p50": 89.81,
"p95": 93.77,
"mean": 89.66241992882561,
"min": 80.88,
"max": 98.81
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1477.0,
"p95": 1552.0,
"mean": 1403.355871886121,
"min": 795.0,
"max": 1672.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 10820.0,
"p95": 14001.0,
"mean": 11487.896797153026,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 95.0,
"p95": 96.0,
"mean": 94.70462633451957,
"min": 84.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 26744.076597355604,
"energy_observed_duration_s": 298.2404053211212,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-host\\telemetry.jsonl",
"samples": 280,
"span_s": 298.37116742134094,
"requested_window": {
"start": 1790062476.7599235,
"stop": 1790062776.7599235
},
"sampling_gap_s": {
"n": 279,
"p50": 1.0699033737182617,
"p95": 1.0833203792572021,
"mean": 1.0694307076033724,
"min": 1.0543808937072754,
"max": 1.1284832954406738
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 280,
"p50": 15320346624.0,
"p95": 15797686272.0,
"mean": 15100751462.4,
"min": 13720989696.0,
"max": 15812558848.0
},
"host_used_bytes": {
"n": 280,
"p50": 52453023744.0,
"p95": 53997940736.0,
"mean": 52672758169.6,
"min": 51960950784.0,
"max": 54052519936.0
},
"host_cpu_percent": {
"n": 280,
"p50": 20.2,
"p95": 28.7,
"mean": 20.69392857142857,
"min": 13.1,
"max": 38.8
},
"gpu_memory_mib": {
"n": 280,
"p50": 14868.0,
"p95": 14893.0,
"mean": 14873.285714285714,
"min": 14776.0,
"max": 15159.0
},
"gpu_temperature_c": {
"n": 280,
"p50": 68.0,
"p95": 70.0,
"mean": 68.31785714285714,
"min": 66.0,
"max": 71.0
},
"gpu_power_w": {
"n": 280,
"p50": 87.4,
"p95": 94.23,
"mean": 87.78892857142857,
"min": 82.19,
"max": 104.67
},
"gpu_sm_clock_mhz": {
"n": 280,
"p50": 1432.0,
"p95": 1545.0,
"mean": 1394.067857142857,
"min": 742.0,
"max": 1605.0
},
"gpu_mem_clock_mhz": {
"n": 280,
"p50": 10820.0,
"p95": 14001.0,
"mean": 11388.035714285714,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 280,
"p50": 95.0,
"p95": 96.0,
"mean": 94.63928571428572,
"min": 82.0,
"max": 98.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 26190.126101810933,
"energy_observed_duration_s": 298.37116742134094,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_60s": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-host\\telemetry.jsonl",
"samples": 55,
"span_s": 58.0647246837616,
"requested_window": {
"start": 1790062716.7599235,
"stop": 1790062776.7599235
},
"sampling_gap_s": {
"n": 54,
"p50": 1.0750997066497803,
"p95": 1.0833203792572021,
"mean": 1.0752726793289185,
"min": 1.064469337463379,
"max": 1.0878071784973145
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 55,
"p50": 15296425984.0,
"p95": 15325220864.0,
"mean": 15288888003.49091,
"min": 15235117056.0,
"max": 15326785536.0
},
"host_used_bytes": {
"n": 55,
"p50": 52477083648.0,
"p95": 52532482048.0,
"mean": 52484621628.509094,
"min": 52446724096.0,
"max": 52538392576.0
},
"host_cpu_percent": {
"n": 55,
"p50": 21.7,
"p95": 29.7,
"mean": 22.863636363636363,
"min": 18.4,
"max": 36.6
},
"gpu_memory_mib": {
"n": 55,
"p50": 14867.0,
"p95": 14881.0,
"mean": 14870.927272727273,
"min": 14859.0,
"max": 14929.0
},
"gpu_temperature_c": {
"n": 55,
"p50": 68.0,
"p95": 69.0,
"mean": 68.30909090909091,
"min": 68.0,
"max": 69.0
},
"gpu_power_w": {
"n": 55,
"p50": 83.93,
"p95": 84.1,
"mean": 83.88927272727273,
"min": 83.24,
"max": 84.21
},
"gpu_sm_clock_mhz": {
"n": 55,
"p50": 1425.0,
"p95": 1462.0,
"mean": 1422.7272727272727,
"min": 1335.0,
"max": 1492.0
},
"gpu_mem_clock_mhz": {
"n": 55,
"p50": 10820.0,
"p95": 10820.0,
"mean": 10877.836363636363,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 55,
"p50": 95.0,
"p95": 96.0,
"mean": 94.9090909090909,
"min": 94.0,
"max": 96.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 4871.0546661305425,
"energy_observed_duration_s": 58.0647246837616,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_request_summary": {
"n": 636,
"rtf_above_one_requests": 0,
"simulated_playback_deficit_requests": 0,
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"scope": "All sustained requests; timing/real-time gates are separate from successful completion."
},
"sustained_request_trends": {
"first_5min": {
"n": 112,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 112,
"p50": 2.683414411963895,
"p95": 2.8355562550132163,
"mean": 2.6950403328825945,
"min": 2.5607825680053793,
"max": 3.16009124397533
},
"ttft_s": {
"n": 112,
"p50": 0.08760976791381836,
"p95": 0.09689497947692871,
"mean": 0.0879168084689549,
"min": 0.07762718200683594,
"max": 0.11411356925964355
},
"decode_tok_per_s": {
"n": 112,
"p50": 48.89259943545705,
"p95": 50.82948720652364,
"mean": 48.75997934521542,
"min": 41.45514880460421,
"max": 51.356495580418965
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_5min": {
"n": 108,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 108,
"p50": 2.7436706610023975,
"p95": 2.8930124340113252,
"mean": 2.7617445574713337,
"min": 2.5760894999839365,
"max": 3.1278614890179597
},
"ttft_s": {
"n": 108,
"p50": 0.09189915657043457,
"p95": 0.10075259208679199,
"mean": 0.0918649744104456,
"min": 0.07878232002258301,
"max": 0.11155438423156738
},
"decode_tok_per_s": {
"n": 108,
"p50": 47.85535853071578,
"p95": 50.64978127024428,
"mean": 47.64494412523459,
"min": 41.82233809777184,
"max": 51.11112403258091
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
},
"last_60s": {
"n": 20,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 20,
"p50": 2.8531201250152662,
"p95": 2.9956124960444868,
"mean": 2.874977746899822,
"min": 2.832491864974145,
"max": 3.1278614890179597
},
"ttft_s": {
"n": 20,
"p50": 0.09533071517944336,
"p95": 0.10180091857910156,
"mean": 0.0961810827255249,
"min": 0.08915209770202637,
"max": 0.11155438423156738
},
"decode_tok_per_s": {
"n": 20,
"p50": 46.02259119862882,
"p95": 46.37814150177049,
"mean": 45.73204792269801,
"min": 41.82233809777184,
"max": 46.40986528698927
},
"total_wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"rtf_total": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"stall_at_ttfa_ms": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
}
}
}
}
},
"spark-cuda-wsl-reliability": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-reliability-host\\telemetry.jsonl",
"samples": 83,
"span_s": 87.8257269859314,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 82,
"p50": 1.071969985961914,
"p95": 1.083503246307373,
"mean": 1.071045451047944,
"min": 1.0575995445251465,
"max": 1.0893840789794922
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 83,
"p50": 20988080128.0,
"p95": 22913978368.0,
"mean": 20948693041.349396,
"min": 19653750784.0,
"max": 23043334144.0
},
"host_used_bytes": {
"n": 83,
"p50": 46785429504.0,
"p95": 48108515328.0,
"mean": 46824816590.650604,
"min": 44730175488.0,
"max": 48119758848.0
},
"host_cpu_percent": {
"n": 83,
"p50": 25.6,
"p95": 37.6,
"mean": 25.997590361445784,
"min": 16.7,
"max": 47.7
},
"gpu_memory_mib": {
"n": 83,
"p50": 10898.0,
"p95": 13231.0,
"mean": 8275.686746987953,
"min": 2052.0,
"max": 13232.0
},
"gpu_temperature_c": {
"n": 83,
"p50": 62.0,
"p95": 63.0,
"mean": 61.45783132530121,
"min": 59.0,
"max": 64.0
},
"gpu_power_w": {
"n": 83,
"p50": 34.75,
"p95": 83.51,
"mean": 40.72216867469879,
"min": 25.2,
"max": 95.7
},
"gpu_sm_clock_mhz": {
"n": 83,
"p50": 1132.0,
"p95": 2017.0,
"mean": 1388.3373493975903,
"min": 877.0,
"max": 2100.0
},
"gpu_mem_clock_mhz": {
"n": 83,
"p50": 9001.0,
"p95": 14001.0,
"mean": 9466.518072289156,
"min": 9001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 83,
"p50": 20.0,
"p95": 95.0,
"mean": 27.650602409638555,
"min": 16.0,
"max": 97.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 3581.5406323730945,
"energy_observed_duration_s": 87.8257269859314,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": null,
"host_launch_to_benchmark_report_start_s": 11.111054182052612,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1330661) INFO 09-22 17:28:50 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 34.719398 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=1330661) INFO 09-22 17:28:55 [core.py:368] init engine (profile, create kv cache, warmup model) took 3.94 s",
"INFO 09-22 17:28:56 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"spark-cuda-wsl-trace": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-trace-host\\telemetry.jsonl",
"samples": 113,
"span_s": 119.62841844558716,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 112,
"p50": 1.065159797668457,
"p95": 1.0797204971313477,
"mean": 1.0681108789784568,
"min": 1.0573580265045166,
"max": 1.1302719116210938
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 113,
"p50": 17488793600.0,
"p95": 20111093760.0,
"mean": 16819074981.380531,
"min": 13674090496.0,
"max": 21769494528.0
},
"host_used_bytes": {
"n": 113,
"p50": 50284716032.0,
"p95": 54063104000.0,
"mean": 50954434650.61947,
"min": 46004015104.0,
"max": 54099419136.0
},
"host_cpu_percent": {
"n": 113,
"p50": 23.4,
"p95": 38.0,
"mean": 24.797345132743363,
"min": 15.3,
"max": 43.9
},
"gpu_memory_mib": {
"n": 113,
"p50": 12044.0,
"p95": 14387.0,
"mean": 10613.893805309735,
"min": 3209.0,
"max": 14417.0
},
"gpu_temperature_c": {
"n": 113,
"p50": 60.0,
"p95": 64.0,
"mean": 59.94690265486726,
"min": 58.0,
"max": 65.0
},
"gpu_power_w": {
"n": 113,
"p50": 34.18,
"p95": 83.07,
"mean": 37.97451327433628,
"min": 31.43,
"max": 96.35
},
"gpu_sm_clock_mhz": {
"n": 113,
"p50": 1087.0,
"p95": 2002.0,
"mean": 1345.8495575221239,
"min": 652.0,
"max": 2197.0
},
"gpu_mem_clock_mhz": {
"n": 113,
"p50": 9001.0,
"p95": 10820.0,
"mean": 9274.575221238938,
"min": 9001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 113,
"p50": 20.0,
"p95": 89.0,
"mean": 24.061946902654867,
"min": 16.0,
"max": 96.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 4547.574654780627,
"energy_observed_duration_s": 119.62841844558716,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 17:13:01 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 1101963"
]
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 60.83823833800852,
"host_launch_to_benchmark_report_start_s": 15.614124298095703,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1101963) INFO 09-22 17:12:13 [gpu_model_runner.py:5471] Model loading took 7.66 GiB memory and 39.550505 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=1101963) INFO 09-22 17:12:17 [core.py:368] init engine (profile, create kv cache, warmup model) took 4.36 s",
"INFO 09-22 17:12:18 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-windows-attribution": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-attribution-host\\telemetry.jsonl",
"samples": 327,
"span_s": 346.45146799087524,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 326,
"p50": 1.0623962879180908,
"p95": 1.0749900341033936,
"mean": 1.062734564389188,
"min": 1.0485868453979492,
"max": 1.1045114994049072
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 327,
"p50": 20984381440.0,
"p95": 24491741184.0,
"mean": 21574773857.076454,
"min": 20470099968.0,
"max": 25840582656.0
},
"host_used_bytes": {
"n": 327,
"p50": 46789128192.0,
"p95": 46973276160.0,
"mean": 46198735774.923546,
"min": 41932926976.0,
"max": 47303409664.0
},
"host_cpu_percent": {
"n": 327,
"p50": 16.3,
"p95": 28.1,
"mean": 17.348318042813453,
"min": 4.2,
"max": 41.8
},
"gpu_memory_mib": {
"n": 327,
"p50": 11459.0,
"p95": 11472.0,
"mean": 9609.021406727828,
"min": 1360.0,
"max": 11733.0
},
"gpu_temperature_c": {
"n": 327,
"p50": 58.0,
"p95": 59.0,
"mean": 57.957186544342505,
"min": 55.0,
"max": 60.0
},
"gpu_power_w": {
"n": 327,
"p50": 32.18,
"p95": 36.38,
"mean": 29.038440366972477,
"min": 13.26,
"max": 47.5
},
"gpu_sm_clock_mhz": {
"n": 327,
"p50": 675.0,
"p95": 1500.0,
"mean": 789.571865443425,
"min": 210.0,
"max": 1867.0
},
"gpu_mem_clock_mhz": {
"n": 327,
"p50": 810.0,
"p95": 810.0,
"mean": 736.9266055045872,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 327,
"p50": 98.0,
"p95": 100.0,
"mean": 74.72782874617737,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 10072.500036509036,
"energy_observed_duration_s": 346.45146799087524,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "existing disk/JIT caches; not cold disk",
"constructor_s": 61.969604700105265,
"host_launch_to_benchmark_report_start_s": 19.921674966812134,
"host_launch_to_first_request_s": 82.20844435691833,
"host_launch_to_first_output_s": 82.9820847568335,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "warmup-medium-1-1",
"length_band": "medium",
"concurrency": 1,
"wall_s": null,
"ttft_s": null,
"total_wall_s": 11.285508499946445,
"ttfa_ms": 773.6403999151662,
"rtf_total": 1.5333571331448974
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=62016) INFO 09-22 18:32:31 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 2.981941 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=62016) INFO 09-22 18:32:33 [monitor.py:53] torch.compile took 0.32 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=62016) INFO 09-22 18:32:47 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=62016) INFO 09-22 18:32:48 [core.py:361] init engine (profile, create kv cache, warmup model) took 16.91 s (compilation: 0.32 s)",
"INFO 09-22 18:32:48 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=13708) INFO 09-22 18:32:54 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 3.482069 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=13708) INFO 09-22 18:32:57 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.14 s",
"INFO 09-22 18:32:57 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-windows": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-host\\telemetry.jsonl",
"samples": 2118,
"span_s": 2263.5735564231873,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 2117,
"p50": 1.067406415939331,
"p95": 1.0880568027496338,
"mean": 1.0692364461139288,
"min": 1.0502521991729736,
"max": 1.6669890880584717
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 2118,
"p50": 30484025344.0,
"p95": 32806387712.0,
"mean": 27893506738.402267,
"min": 16798679040.0,
"max": 34461278208.0
},
"host_used_bytes": {
"n": 2118,
"p50": 37289185280.0,
"p95": 50826862592.0,
"mean": 39880002893.59773,
"min": 33312231424.0,
"max": 50974830592.0
},
"host_cpu_percent": {
"n": 2118,
"p50": 36.5,
"p95": 51.1,
"mean": 35.88347497639282,
"min": 6.8,
"max": 74.9
},
"gpu_memory_mib": {
"n": 2118,
"p50": 12600.0,
"p95": 13036.0,
"mean": 12206.915958451369,
"min": 1665.0,
"max": 13130.0
},
"gpu_temperature_c": {
"n": 2118,
"p50": 68.0,
"p95": 72.0,
"mean": 66.58687440982058,
"min": 55.0,
"max": 73.0
},
"gpu_power_w": {
"n": 2118,
"p50": 83.93,
"p95": 98.71,
"mean": 77.1445136921624,
"min": 13.76,
"max": 129.97
},
"gpu_sm_clock_mhz": {
"n": 2118,
"p50": 1635.0,
"p95": 1837.0,
"mean": 1513.040132200189,
"min": 180.0,
"max": 2535.0
},
"gpu_mem_clock_mhz": {
"n": 2118,
"p50": 14001.0,
"p95": 14001.0,
"mean": 12019.57223796034,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 2118,
"p50": 90.0,
"p95": 98.0,
"mean": 82.03493862134088,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 174771.95130131126,
"energy_observed_duration_s": 2263.5735564231873,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 3214,
"examples": [
"WARNING 09-22 17:38:18 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req warmup-short-1-1-85f54a94 at stage-0 (known reqs: [])",
"WARNING 09-22 17:38:18 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req warmup-short-1-1-85f54a94 at stage-0 (known reqs: [])",
"WARNING 09-22 17:38:18 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req warmup-short-1-1-85f54a94 at stage-0 (known reqs: [])",
"WARNING 09-22 18:10:09 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req sustained-medium-1-1264-afa3b073 at stage-0 (known reqs: ['sustained-medium-1-1265-a676651e'])",
"WARNING 09-22 18:10:11 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req sustained-medium-1-1266-aa2c8b1e at stage-0 (known reqs: [])",
"WARNING 09-22 18:10:11 [orchestrator.py:1513] [Orchestrator] Dropping output for unknown req sustained-medium-1-1266-aa2c8b1e at stage-0 (known reqs: [])"
]
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "existing disk/JIT caches; not cold disk",
"constructor_s": 79.79245940002147,
"host_launch_to_benchmark_report_start_s": 24.507885694503784,
"host_launch_to_first_request_s": 104.82267737388611,
"host_launch_to_first_output_s": 105.72827107389458,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "warmup-short-1-1",
"length_band": "short",
"concurrency": 1,
"wall_s": null,
"ttft_s": null,
"total_wall_s": 1.373490900034085,
"ttfa_ms": 905.5937000084668,
"rtf_total": 0.4187472256201479
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=65260) INFO 09-22 17:37:38 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 3.965538 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=65260) INFO 09-22 17:37:40 [monitor.py:53] torch.compile took 0.40 s in total",
"(StageEngineCoreProc_stage1_replica0 pid=63504) INFO 09-22 17:37:48 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 3.218381 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=65260) INFO 09-22 17:38:08 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=65260) INFO 09-22 17:38:09 [core.py:361] init engine (profile, create kv cache, warmup model) took 29.94 s (compilation: 0.40 s)",
"INFO 09-22 17:38:09 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=63504) INFO 09-22 17:38:11 [core.py:368] init engine (profile, create kv cache, warmup model) took 22.87 s",
"INFO 09-22 17:38:12 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
},
"sustained": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-host\\telemetry.jsonl",
"samples": 1688,
"span_s": 1803.7699987888336,
"requested_window": {
"start": 1790070233.2147033,
"stop": 1790072037.5126588
},
"sampling_gap_s": {
"n": 1687,
"p50": 1.0670323371887207,
"p95": 1.0887537002563477,
"mean": 1.0692175452215966,
"min": 1.0502521991729736,
"max": 1.6669890880584717
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 1688,
"p50": 30733127680.0,
"p95": 32820871168.0,
"mean": 30290101929.85782,
"min": 17072979968.0,
"max": 33141321728.0
},
"host_used_bytes": {
"n": 1688,
"p50": 37035700224.0,
"p95": 50068860928.0,
"mean": 37483407702.14218,
"min": 34632187904.0,
"max": 50700529664.0
},
"host_cpu_percent": {
"n": 1688,
"p50": 36.4,
"p95": 48.4,
"mean": 35.68329383886256,
"min": 10.4,
"max": 65.3
},
"gpu_memory_mib": {
"n": 1688,
"p50": 12808.0,
"p95": 13044.0,
"mean": 12669.649881516587,
"min": 12117.0,
"max": 13130.0
},
"gpu_temperature_c": {
"n": 1688,
"p50": 68.0,
"p95": 71.0,
"mean": 66.64395734597156,
"min": 55.0,
"max": 73.0
},
"gpu_power_w": {
"n": 1688,
"p50": 83.77,
"p95": 98.24,
"mean": 78.57734597156399,
"min": 21.97,
"max": 129.97
},
"gpu_sm_clock_mhz": {
"n": 1688,
"p50": 1642.0,
"p95": 1830.0,
"mean": 1535.0017772511849,
"min": 180.0,
"max": 2032.0
},
"gpu_mem_clock_mhz": {
"n": 1688,
"p50": 14001.0,
"p95": 14001.0,
"mean": 12325.7422985782,
"min": 810.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 1688,
"p50": 90.0,
"p95": 98.0,
"mean": 85.48104265402844,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 141821.73650193692,
"energy_observed_duration_s": 1803.7699987888336,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_first_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-host\\telemetry.jsonl",
"samples": 281,
"span_s": 298.63769698143005,
"requested_window": {
"start": 1790070233.2147033,
"stop": 1790070533.2147033
},
"sampling_gap_s": {
"n": 280,
"p50": 1.0651309490203857,
"p95": 1.078244686126709,
"mean": 1.0665632035051074,
"min": 1.0535480976104736,
"max": 1.1121037006378174
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 32738803712.0,
"p95": 33080823808.0,
"mean": 27361576457.11032,
"min": 17072979968.0,
"max": 33141321728.0
},
"host_used_bytes": {
"n": 281,
"p50": 35034705920.0,
"p95": 50654482432.0,
"mean": 40411933174.88968,
"min": 34632187904.0,
"max": 50700529664.0
},
"host_cpu_percent": {
"n": 281,
"p50": 38.1,
"p95": 48.1,
"mean": 38.33238434163701,
"min": 24.2,
"max": 54.0
},
"gpu_memory_mib": {
"n": 281,
"p50": 12229.0,
"p95": 12241.0,
"mean": 12195.192170818505,
"min": 12117.0,
"max": 12253.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 66.0,
"p95": 71.0,
"mean": 67.02846975088968,
"min": 63.0,
"max": 72.0
},
"gpu_power_w": {
"n": 281,
"p50": 86.17,
"p95": 98.23,
"mean": 85.15330960854094,
"min": 26.82,
"max": 129.97
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1657.0,
"p95": 1837.0,
"mean": 1632.4733096085408,
"min": 682.0,
"max": 2025.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 14001.0,
"p95": 14001.0,
"mean": 13989.67971530249,
"min": 10820.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 89.0,
"p95": 92.0,
"mean": 81.8185053380783,
"min": 0.0,
"max": 93.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 25416.702287727596,
"energy_observed_duration_s": 298.63769698143005,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-host\\telemetry.jsonl",
"samples": 281,
"span_s": 298.9493010044098,
"requested_window": {
"start": 1790071737.5126588,
"stop": 1790072037.5126588
},
"sampling_gap_s": {
"n": 280,
"p50": 1.064634084701538,
"p95": 1.0855522155761719,
"mean": 1.0676760750157492,
"min": 1.0502521991729736,
"max": 1.6669890880584717
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 30231056384.0,
"p95": 31668371456.0,
"mean": 30295986117.69395,
"min": 28525268992.0,
"max": 31725813760.0
},
"host_used_bytes": {
"n": 281,
"p50": 37542453248.0,
"p95": 39208062976.0,
"mean": 37477523514.30605,
"min": 36047695872.0,
"max": 39248240640.0
},
"host_cpu_percent": {
"n": 281,
"p50": 19.6,
"p95": 40.4,
"mean": 22.7779359430605,
"min": 10.4,
"max": 65.3
},
"gpu_memory_mib": {
"n": 281,
"p50": 13002.0,
"p95": 13051.0,
"mean": 13010.61921708185,
"min": 12994.0,
"max": 13087.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 57.0,
"p95": 67.0,
"mean": 58.84341637010676,
"min": 55.0,
"max": 68.0
},
"gpu_power_w": {
"n": 281,
"p50": 34.18,
"p95": 83.87,
"mean": 45.18540925266904,
"min": 21.97,
"max": 85.31
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1132.0,
"p95": 1717.0,
"mean": 1061.2313167259786,
"min": 180.0,
"max": 1972.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 810.0,
"p95": 14001.0,
"mean": 4050.733096085409,
"min": 810.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 98.0,
"p95": 100.0,
"mean": 93.92170818505338,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 13542.867505235672,
"energy_observed_duration_s": 298.9493010044098,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_60s": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-host\\telemetry.jsonl",
"samples": 57,
"span_s": 59.53130531311035,
"requested_window": {
"start": 1790071977.5126588,
"stop": 1790072037.5126588
},
"sampling_gap_s": {
"n": 56,
"p50": 1.0629546642303467,
"p95": 1.0778369903564453,
"mean": 1.0630590234483992,
"min": 1.0502521991729736,
"max": 1.0861613750457764
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 57,
"p50": 31542673408.0,
"p95": 31610490880.0,
"mean": 31537425425.964912,
"min": 31363129344.0,
"max": 31627063296.0
},
"host_used_bytes": {
"n": 57,
"p50": 36230836224.0,
"p95": 36364054528.0,
"mean": 36236084206.03509,
"min": 36146446336.0,
"max": 36410380288.0
},
"host_cpu_percent": {
"n": 57,
"p50": 16.3,
"p95": 24.6,
"mean": 17.166666666666668,
"min": 10.9,
"max": 34.9
},
"gpu_memory_mib": {
"n": 57,
"p50": 13002.0,
"p95": 13015.0,
"mean": 13003.298245614034,
"min": 13002.0,
"max": 13015.0
},
"gpu_temperature_c": {
"n": 57,
"p50": 57.0,
"p95": 57.0,
"mean": 56.91228070175438,
"min": 56.0,
"max": 58.0
},
"gpu_power_w": {
"n": 57,
"p50": 33.62,
"p95": 36.18,
"mean": 33.46859649122807,
"min": 22.5,
"max": 37.05
},
"gpu_sm_clock_mhz": {
"n": 57,
"p50": 832.0,
"p95": 1387.0,
"mean": 903.1578947368421,
"min": 435.0,
"max": 1410.0
},
"gpu_mem_clock_mhz": {
"n": 57,
"p50": 810.0,
"p95": 810.0,
"mean": 810.0,
"min": 810.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 57,
"p50": 99.0,
"p95": 100.0,
"mean": 97.12280701754386,
"min": 3.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 1999.857006803751,
"energy_observed_duration_s": 59.53130531311035,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_request_summary": {
"n": 1087,
"rtf_above_one_requests": 21,
"simulated_playback_deficit_requests": 98,
"rtf_total": {
"n": 1087,
"p50": 0.185484508427875,
"p95": 0.20647447442818453,
"mean": 0.20844494376110967,
"min": 0.16350789286661893,
"max": 1.408214111832326
},
"scope": "All sustained requests; timing/real-time gates are separate from successful completion."
},
"sustained_request_trends": {
"first_5min": {
"n": 208,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 208,
"p50": 1.4048798000440001,
"p95": 1.7023280001012608,
"mean": 1.4403136653905118,
"min": 1.2072327000787482,
"max": 1.7842953000217676
},
"ttfa_ms": {
"n": 208,
"p50": 99.54510000534356,
"p95": 271.77320001646876,
"mean": 131.82270240605038,
"min": 78.2809000229463,
"max": 286.0235000262037
},
"rtf_total": {
"n": 208,
"p50": 0.1775149999957648,
"p95": 0.1985024107258995,
"mean": 0.18031473771197046,
"min": 0.16350789286661893,
"max": 0.21162559065627506
},
"stall_at_ttfa_ms": {
"n": 208,
"p50": 0.0,
"p95": 42.983299968764186,
"mean": 4.718476443645509,
"min": 0.0,
"max": 73.79150009714067
}
}
},
"last_5min": {
"n": 71,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 71,
"p50": 1.5336161999730393,
"p95": 11.445048100082204,
"mean": 4.20997989436769,
"min": 1.2494406000478193,
"max": 12.033524899976328
},
"ttfa_ms": {
"n": 71,
"p50": 98.11000002082437,
"p95": 378.7460000021383,
"mean": 174.3538014125079,
"min": 61.44930003210902,
"max": 400.6056999787688
},
"rtf_total": {
"n": 71,
"p50": 0.19243645160371356,
"p95": 1.3793616625043796,
"mean": 0.5401730472838914,
"min": 0.17824191011467508,
"max": 1.408214111832326
},
"stall_at_ttfa_ms": {
"n": 71,
"p50": 0.0,
"p95": 4263.7755000265315,
"mean": 1190.2638506979852,
"min": 0.0,
"max": 4650.390599993989
}
}
},
"last_60s": {
"n": 5,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 5,
"p50": 10.42487240000628,
"p95": 11.445048100082204,
"mean": 10.33818040001206,
"min": 9.336497799959034,
"max": 11.445048100082204
},
"ttfa_ms": {
"n": 5,
"p50": 346.3353000115603,
"p95": 349.6311000781134,
"mean": 342.5582800293341,
"min": 334.03979998547584,
"max": 349.6311000781134
},
"rtf_total": {
"n": 5,
"p50": 1.3162717676775606,
"p95": 1.3370383294488555,
"mean": 1.3184726956937667,
"min": 1.3061821634647248,
"max": 1.3370383294488555
},
"stall_at_ttfa_ms": {
"n": 5,
"p50": 3674.5423999615014,
"p95": 4371.076499950141,
"mean": 3861.0308199655265,
"min": 3561.7392999352887,
"max": 4371.076499950141
}
}
}
}
},
"tts-cuda-windows-reliability": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-reliability-host\\telemetry.jsonl",
"samples": 95,
"span_s": 99.86338472366333,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 94,
"p50": 1.061262845993042,
"p95": 1.0735931396484375,
"mean": 1.062376433230461,
"min": 1.0483684539794922,
"max": 1.0836434364318848
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 95,
"p50": 32130404352.0,
"p95": 34583187456.0,
"mean": 31904378190.14737,
"min": 29141966848.0,
"max": 35300249600.0
},
"host_used_bytes": {
"n": 95,
"p50": 35643105280.0,
"p95": 38614671360.0,
"mean": 35869131441.85263,
"min": 32473260032.0,
"max": 38631542784.0
},
"host_cpu_percent": {
"n": 95,
"p50": 12.2,
"p95": 24.9,
"mean": 13.377894736842107,
"min": 5.1,
"max": 36.0
},
"gpu_memory_mib": {
"n": 95,
"p50": 5290.0,
"p95": 13046.0,
"mean": 7040.073684210526,
"min": 2905.0,
"max": 13126.0
},
"gpu_temperature_c": {
"n": 95,
"p50": 56.0,
"p95": 57.0,
"mean": 55.578947368421055,
"min": 55.0,
"max": 57.0
},
"gpu_power_w": {
"n": 95,
"p50": 14.84,
"p95": 33.66,
"mean": 19.65915789473684,
"min": 12.9,
"max": 37.67
},
"gpu_sm_clock_mhz": {
"n": 95,
"p50": 390.0,
"p95": 1800.0,
"mean": 603.3789473684211,
"min": 187.0,
"max": 2040.0
},
"gpu_mem_clock_mhz": {
"n": 95,
"p50": 405.0,
"p95": 810.0,
"mean": 571.2631578947369,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 95,
"p50": 1.0,
"p95": 99.0,
"mean": 24.894736842105264,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 1969.2352614307404,
"energy_observed_duration_s": 99.86338472366333,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": null,
"host_launch_to_benchmark_report_start_s": 16.83585023880005,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=64316) INFO 09-22 18:15:05 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 2.209315 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=64316) INFO 09-22 18:15:06 [monitor.py:53] torch.compile took 0.27 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=64316) INFO 09-22 18:15:20 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=64316) INFO 09-22 18:15:20 [core.py:361] init engine (profile, create kv cache, warmup model) took 15.22 s (compilation: 0.27 s)",
"INFO 09-22 18:15:21 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=60788) INFO 09-22 18:15:26 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 3.208076 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=60788) INFO 09-22 18:15:28 [core.py:368] init engine (profile, create kv cache, warmup model) took 1.98 s",
"INFO 09-22 18:15:29 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-windows-trace": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-trace-host\\telemetry.jsonl",
"samples": 137,
"span_s": 144.3495752811432,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 136,
"p50": 1.0617845058441162,
"p95": 1.07218599319458,
"mean": 1.0613939358907587,
"min": 1.0484578609466553,
"max": 1.105041742324829
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 137,
"p50": 18266259456.0,
"p95": 21369876480.0,
"mean": 18746558508.846714,
"min": 16364032000.0,
"max": 22450339840.0
},
"host_used_bytes": {
"n": 137,
"p50": 49507250176.0,
"p95": 51397099520.0,
"mean": 49026951123.15328,
"min": 45323169792.0,
"max": 51409477632.0
},
"host_cpu_percent": {
"n": 137,
"p50": 13.0,
"p95": 22.0,
"mean": 13.761313868613138,
"min": 3.7,
"max": 41.2
},
"gpu_memory_mib": {
"n": 137,
"p50": 11317.0,
"p95": 11514.0,
"mean": 7281.1021897810215,
"min": 1284.0,
"max": 11518.0
},
"gpu_temperature_c": {
"n": 137,
"p50": 56.0,
"p95": 57.0,
"mean": 56.18978102189781,
"min": 55.0,
"max": 58.0
},
"gpu_power_w": {
"n": 137,
"p50": 15.73,
"p95": 36.02,
"mean": 21.416131386861316,
"min": 13.17,
"max": 41.78
},
"gpu_sm_clock_mhz": {
"n": 137,
"p50": 405.0,
"p95": 1710.0,
"mean": 618.5036496350365,
"min": 187.0,
"max": 2017.0
},
"gpu_mem_clock_mhz": {
"n": 137,
"p50": 405.0,
"p95": 810.0,
"mean": 582.3722627737226,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 137,
"p50": 1.0,
"p95": 99.0,
"mean": 32.0,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 3099.191924674511,
"energy_observed_duration_s": 144.3495752811432,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 56.888828900060616,
"host_launch_to_benchmark_report_start_s": 21.322706699371338,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=48636) INFO 09-22 18:58:03 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 3.090256 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=48636) INFO 09-22 18:58:05 [monitor.py:53] torch.compile took 0.28 s in total",
"(StageEngineCoreProc_stage1_replica0 pid=59532) INFO 09-22 18:58:11 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 3.415938 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=48636) INFO 09-22 18:58:25 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=48636) INFO 09-22 18:58:26 [core.py:361] init engine (profile, create kv cache, warmup model) took 22.56 s (compilation: 0.28 s)",
"INFO 09-22 18:58:26 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=59532) INFO 09-22 18:58:28 [core.py:368] init engine (profile, create kv cache, warmup model) took 15.76 s",
"INFO 09-22 18:58:28 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-wsl-attribution": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-attribution-host\\telemetry.jsonl",
"samples": 1126,
"span_s": 1207.9842503070831,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 1125,
"p50": 1.0627398490905762,
"p95": 1.0752325057983398,
"mean": 1.0737637780507405,
"min": 1.046626091003418,
"max": 3.2002604007720947
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 1126,
"p50": 5813039104.0,
"p95": 17809747968.0,
"mean": 8238610944.909414,
"min": 54951936.0,
"max": 23584870400.0
},
"host_used_bytes": {
"n": 1126,
"p50": 61960175616.0,
"p95": 64520925184.0,
"mean": 59534898687.09058,
"min": 44188639232.0,
"max": 67718557696.0
},
"host_cpu_percent": {
"n": 1126,
"p50": 13.7,
"p95": 23.9,
"mean": 14.86918294849023,
"min": 4.0,
"max": 55.1
},
"gpu_memory_mib": {
"n": 1126,
"p50": 23862.0,
"p95": 23875.0,
"mean": 20226.72735346359,
"min": 1304.0,
"max": 23932.0
},
"gpu_temperature_c": {
"n": 1126,
"p50": 59.0,
"p95": 63.0,
"mean": 59.52131438721137,
"min": 54.0,
"max": 63.0
},
"gpu_power_w": {
"n": 1126,
"p50": 32.95,
"p95": 35.62,
"mean": 32.29695381882771,
"min": 13.1,
"max": 40.02
},
"gpu_sm_clock_mhz": {
"n": 1126,
"p50": 1665.0,
"p95": 1695.0,
"mean": 1402.6056838365896,
"min": 180.0,
"max": 2092.0
},
"gpu_mem_clock_mhz": {
"n": 1126,
"p50": 810.0,
"p95": 810.0,
"mean": 796.6918294849023,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 1126,
"p50": 100.0,
"p95": 100.0,
"mean": 95.42095914742451,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 38994.17466940165,
"energy_observed_duration_s": 1207.9842503070831,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "existing disk/JIT caches; not cold disk",
"constructor_s": 917.268663698982,
"host_launch_to_benchmark_report_start_s": 8.934701681137085,
"host_launch_to_first_request_s": 926.4748594760895,
"host_launch_to_first_output_s": 929.7797479120782,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "warmup-medium-1-1",
"length_band": "medium",
"concurrency": 1,
"wall_s": null,
"ttft_s": null,
"total_wall_s": 16.85265357902972,
"ttfa_ms": 3304.888435988687,
"rtf_total": 1.987341223942184
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1628534) INFO 09-22 19:00:53 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 3.103314 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=1628537) INFO 09-22 19:00:54 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 4.010357 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=1628534) INFO 09-22 19:00:55 [monitor.py:53] torch.compile took 0.20 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=1628534) INFO 09-22 19:01:20 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=1628534) INFO 09-22 19:01:22 [core.py:361] init engine (profile, create kv cache, warmup model) took 28.40 s (compilation: 0.20 s)",
"INFO 09-22 19:01:22 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=1628537) INFO 09-22 19:15:48 [core.py:368] init engine (profile, create kv cache, warmup model) took 894.25 s",
"INFO 09-22 19:15:49 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-wsl": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-host\\telemetry.jsonl",
"samples": 2315,
"span_s": 2517.7729234695435,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 2314,
"p50": 1.0670559406280518,
"p95": 1.0898723602294922,
"mean": 1.088060900375775,
"min": 1.050445318222046,
"max": 4.437119960784912
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 2315,
"p50": 20395335680.0,
"p95": 26196111360.0,
"mean": 19348629692.433693,
"min": 24514560.0,
"max": 26777497600.0
},
"host_used_bytes": {
"n": 2315,
"p50": 47378173952.0,
"p95": 65955311616.0,
"mean": 48424879939.56631,
"min": 40996012032.0,
"max": 67748995072.0
},
"host_cpu_percent": {
"n": 2315,
"p50": 25.8,
"p95": 38.2,
"mean": 26.823239740820735,
"min": 9.3,
"max": 73.0
},
"gpu_memory_mib": {
"n": 2315,
"p50": 12404.0,
"p95": 23547.0,
"mean": 13255.07818574514,
"min": 2454.0,
"max": 23892.0
},
"gpu_temperature_c": {
"n": 2315,
"p50": 68.0,
"p95": 71.0,
"mean": 67.96457883369331,
"min": 56.0,
"max": 73.0
},
"gpu_power_w": {
"n": 2315,
"p50": 83.44,
"p95": 92.58,
"mean": 81.39983585313175,
"min": 19.57,
"max": 115.96
},
"gpu_sm_clock_mhz": {
"n": 2315,
"p50": 1567.0,
"p95": 2752.0,
"mean": 1646.3771058315335,
"min": 180.0,
"max": 2782.0
},
"gpu_mem_clock_mhz": {
"n": 2315,
"p50": 14001.0,
"p95": 14001.0,
"mean": 13684.360691144708,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 2315,
"p50": 90.0,
"p95": 100.0,
"mean": 82.83066954643628,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 203279.88765108466,
"energy_observed_duration_s": 2517.7729234695435,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 2,
"examples": [
"WARNING 09-22 16:21:58 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 651158",
"WARNING 09-22 16:22:03 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage1_replica0 pid 651161"
]
}
}
},
"startup": {
"cache_condition": "existing disk/JIT caches; not cold disk",
"constructor_s": 345.16766462504165,
"host_launch_to_benchmark_report_start_s": 11.228320837020874,
"host_launch_to_first_request_s": 356.7392716407776,
"host_launch_to_first_output_s": 358.03673665074166,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": {
"request_id": "warmup-short-1-1",
"length_band": "short",
"concurrency": 1,
"wall_s": null,
"ttft_s": null,
"total_wall_s": 1.8720393629628234,
"ttfa_ms": 1297.4650099640712,
"rtf_total": 0.570743708220373
},
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=651158) INFO 09-22 15:40:47 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 3.392905 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=651161) INFO 09-22 15:40:51 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 6.873289 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=651158) INFO 09-22 15:41:19 [monitor.py:53] torch.compile took 26.99 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=651158) INFO 09-22 15:42:49 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=651158) INFO 09-22 15:42:51 [core.py:361] init engine (profile, create kv cache, warmup model) took 123.75 s (compilation: 26.99 s)",
"INFO 09-22 15:42:51 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=651161) INFO 09-22 15:46:07 [core.py:368] init engine (profile, create kv cache, warmup model) took 315.73 s",
"INFO 09-22 15:46:07 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
},
"sustained": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-host\\telemetry.jsonl",
"samples": 1688,
"span_s": 1799.8631873130798,
"requested_window": {
"start": 1790063509.4051945,
"stop": 1790065310.5886679
},
"sampling_gap_s": {
"n": 1687,
"p50": 1.0656282901763916,
"p95": 1.0789265632629395,
"mean": 1.0669017115074568,
"min": 1.0528502464294434,
"max": 1.119377851486206
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 1688,
"p50": 21400948736.0,
"p95": 26312826880.0,
"mean": 22341723973.1564,
"min": 19646988288.0,
"max": 26777497600.0
},
"host_used_bytes": {
"n": 1688,
"p50": 46370131968.0,
"p95": 47873781760.0,
"mean": 45431785658.843605,
"min": 40996012032.0,
"max": 48126521344.0
},
"host_cpu_percent": {
"n": 1688,
"p50": 25.7,
"p95": 35.4,
"mean": 26.516291469194314,
"min": 18.1,
"max": 51.3
},
"gpu_memory_mib": {
"n": 1688,
"p50": 12401.0,
"p95": 12547.0,
"mean": 12409.829383886256,
"min": 12200.0,
"max": 12822.0
},
"gpu_temperature_c": {
"n": 1688,
"p50": 68.0,
"p95": 71.0,
"mean": 68.07582938388626,
"min": 65.0,
"max": 73.0
},
"gpu_power_w": {
"n": 1688,
"p50": 83.51,
"p95": 92.6,
"mean": 84.38763033175356,
"min": 65.6,
"max": 105.05
},
"gpu_sm_clock_mhz": {
"n": 1688,
"p50": 1567.0,
"p95": 1695.0,
"mean": 1574.7659952606634,
"min": 1320.0,
"max": 1882.0
},
"gpu_mem_clock_mhz": {
"n": 1688,
"p50": 14001.0,
"p95": 14001.0,
"mean": 14001.0,
"min": 14001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 1688,
"p50": 89.0,
"p95": 92.0,
"mean": 85.09241706161137,
"min": 47.0,
"max": 95.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 151886.36985161543,
"energy_observed_duration_s": 1799.8631873130798,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_first_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-host\\telemetry.jsonl",
"samples": 281,
"span_s": 299.6708698272705,
"requested_window": {
"start": 1790063509.4051945,
"stop": 1790063809.4051945
},
"sampling_gap_s": {
"n": 280,
"p50": 1.0701661109924316,
"p95": 1.0823314189910889,
"mean": 1.0702531065259662,
"min": 1.056560754776001,
"max": 1.1100640296936035
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 20247449600.0,
"p95": 20378746880.0,
"mean": 20207307852.52669,
"min": 19672371200.0,
"max": 20400398336.0
},
"host_used_bytes": {
"n": 281,
"p50": 47526060032.0,
"p95": 47753347072.0,
"mean": 47566201779.47331,
"min": 47373111296.0,
"max": 48101138432.0
},
"host_cpu_percent": {
"n": 281,
"p50": 26.4,
"p95": 37.5,
"mean": 27.135231316725978,
"min": 19.9,
"max": 47.9
},
"gpu_memory_mib": {
"n": 281,
"p50": 12370.0,
"p95": 12487.0,
"mean": 12377.170818505338,
"min": 12261.0,
"max": 12749.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 70.0,
"p95": 71.0,
"mean": 69.1459074733096,
"min": 66.0,
"max": 72.0
},
"gpu_power_w": {
"n": 281,
"p50": 83.51,
"p95": 93.02,
"mean": 85.3820640569395,
"min": 74.88,
"max": 105.05
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1567.0,
"p95": 1710.0,
"mean": 1574.3879003558718,
"min": 1387.0,
"max": 1792.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 14001.0,
"p95": 14001.0,
"mean": 14001.0,
"min": 14001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 89.0,
"p95": 92.0,
"mean": 84.33807829181495,
"min": 62.0,
"max": 94.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 25582.757762696743,
"energy_observed_duration_s": 299.6708698272705,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_5min": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-host\\telemetry.jsonl",
"samples": 281,
"span_s": 298.24036955833435,
"requested_window": {
"start": 1790065010.5886679,
"stop": 1790065310.5886679
},
"sampling_gap_s": {
"n": 280,
"p50": 1.0640408992767334,
"p95": 1.0769662857055664,
"mean": 1.0651441769940513,
"min": 1.0542213916778564,
"max": 1.1047275066375732
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 281,
"p50": 26111430656.0,
"p95": 26691563520.0,
"mean": 26143453781.637012,
"min": 25707888640.0,
"max": 26777497600.0
},
"host_used_bytes": {
"n": 281,
"p50": 41662078976.0,
"p95": 41981423616.0,
"mean": 41630055850.36299,
"min": 40996012032.0,
"max": 42065620992.0
},
"host_cpu_percent": {
"n": 281,
"p50": 28.3,
"p95": 35.9,
"mean": 28.706049822064056,
"min": 21.4,
"max": 41.9
},
"gpu_memory_mib": {
"n": 281,
"p50": 12462.0,
"p95": 12600.0,
"mean": 12487.23487544484,
"min": 12441.0,
"max": 12747.0
},
"gpu_temperature_c": {
"n": 281,
"p50": 66.0,
"p95": 69.0,
"mean": 66.50889679715303,
"min": 65.0,
"max": 70.0
},
"gpu_power_w": {
"n": 281,
"p50": 83.4,
"p95": 84.19,
"mean": 83.45252669039147,
"min": 77.87,
"max": 96.59
},
"gpu_sm_clock_mhz": {
"n": 281,
"p50": 1575.0,
"p95": 1657.0,
"mean": 1575.7580071174377,
"min": 1425.0,
"max": 1777.0
},
"gpu_mem_clock_mhz": {
"n": 281,
"p50": 14001.0,
"p95": 14001.0,
"mean": 14001.0,
"min": 14001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 281,
"p50": 90.0,
"p95": 93.0,
"mean": 85.41637010676156,
"min": 56.0,
"max": 95.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 24885.05900680423,
"energy_observed_duration_s": 298.24036955833435,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_last_60s": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-host\\telemetry.jsonl",
"samples": 56,
"span_s": 58.52069449424744,
"requested_window": {
"start": 1790065250.5886679,
"stop": 1790065310.5886679
},
"sampling_gap_s": {
"n": 55,
"p50": 1.0627195835113525,
"p95": 1.074115514755249,
"mean": 1.0640126271681352,
"min": 1.0555412769317627,
"max": 1.0819242000579834
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 56,
"p50": 26638016512.0,
"p95": 26762883072.0,
"mean": 26410268233.142857,
"min": 25708445696.0,
"max": 26777497600.0
},
"host_used_bytes": {
"n": 56,
"p50": 41135206400.0,
"p95": 42046423040.0,
"mean": 41363241398.85714,
"min": 40996012032.0,
"max": 42065063936.0
},
"host_cpu_percent": {
"n": 56,
"p50": 30.5,
"p95": 38.9,
"mean": 30.989285714285717,
"min": 25.0,
"max": 41.9
},
"gpu_memory_mib": {
"n": 56,
"p50": 12534.0,
"p95": 12615.0,
"mean": 12535.482142857143,
"min": 12471.0,
"max": 12626.0
},
"gpu_temperature_c": {
"n": 56,
"p50": 66.0,
"p95": 69.0,
"mean": 66.66071428571429,
"min": 65.0,
"max": 70.0
},
"gpu_power_w": {
"n": 56,
"p50": 83.38,
"p95": 83.96,
"mean": 83.27321428571429,
"min": 81.75,
"max": 84.15
},
"gpu_sm_clock_mhz": {
"n": 56,
"p50": 1575.0,
"p95": 1657.0,
"mean": 1576.1964285714287,
"min": 1477.0,
"max": 1740.0
},
"gpu_mem_clock_mhz": {
"n": 56,
"p50": 14001.0,
"p95": 14001.0,
"mean": 14001.0,
"min": 14001.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 56,
"p50": 89.0,
"p95": 92.0,
"mean": 85.10714285714286,
"min": 70.0,
"max": 93.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 4873.295795217752,
"energy_observed_duration_s": 58.52069449424744,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"sustained_request_summary": {
"n": 1269,
"rtf_above_one_requests": 0,
"simulated_playback_deficit_requests": 0,
"rtf_total": {
"n": 1269,
"p50": 0.17831764664690664,
"p95": 0.18401261262518798,
"mean": 0.1777871774853687,
"min": 0.16280862525621032,
"max": 0.1927966961627342
},
"scope": "All sustained requests; timing/real-time gates are separate from successful completion."
},
"sustained_request_trends": {
"first_5min": {
"n": 215,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 215,
"p50": 1.3937259759986773,
"p95": 1.602686298021581,
"mean": 1.3952063412508429,
"min": 1.0853738259756938,
"max": 1.7856557319755666
},
"ttfa_ms": {
"n": 215,
"p50": 66.24030799139291,
"p95": 111.39357404317707,
"mean": 72.56843104260042,
"min": 55.547748983372,
"max": 136.65197300724685
},
"rtf_total": {
"n": 215,
"p50": 0.17736189186604676,
"p95": 0.18328674414078705,
"mean": 0.1761833914376344,
"min": 0.16280862525621032,
"max": 0.1918485598922656
},
"stall_at_ttfa_ms": {
"n": 215,
"p50": 0.0,
"p95": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
"last_5min": {
"n": 207,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 207,
"p50": 1.430598111008294,
"p95": 1.6158843949669972,
"mean": 1.4406145231706622,
"min": 1.2058070889906958,
"max": 2.1392868050024845
},
"ttfa_ms": {
"n": 207,
"p50": 65.73405704693869,
"p95": 98.41795702232048,
"mean": 70.32011596655815,
"min": 56.837406009435654,
"max": 136.01501897210255
},
"rtf_total": {
"n": 207,
"p50": 0.17952092796607308,
"p95": 0.18417310503022946,
"mean": 0.17954250247177336,
"min": 0.16970314527700087,
"max": 0.18696815252738777
},
"stall_at_ttfa_ms": {
"n": 207,
"p50": 0.0,
"p95": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
"last_60s": {
"n": 42,
"window_basis": "request submission time",
"metrics": {
"wall_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"ttft_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"decode_tok_per_s": {
"n": 0,
"p50": null,
"p95": null,
"mean": null
},
"total_wall_s": {
"n": 42,
"p50": 1.4264708300470375,
"p95": 1.5752009449643083,
"mean": 1.4246827097141206,
"min": 1.2058070889906958,
"max": 1.7121564430417493
},
"ttfa_ms": {
"n": 42,
"p50": 66.02344202110544,
"p95": 93.17998704500496,
"mean": 69.89174404796879,
"min": 57.876036036759615,
"max": 103.97763003129512
},
"rtf_total": {
"n": 42,
"p50": 0.1796128481219057,
"p95": 0.18469743946308187,
"mean": 0.18009365249732845,
"min": 0.17545585167719072,
"max": 0.18562177776689887
},
"stall_at_ttfa_ms": {
"n": 42,
"p50": 0.0,
"p95": 0.0,
"mean": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
}
},
"tts-cuda-wsl-reliability": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-reliability-host\\telemetry.jsonl",
"samples": 217,
"span_s": 233.3380994796753,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 216,
"p50": 1.0771429538726807,
"p95": 1.1066455841064453,
"mean": 1.0802689790725708,
"min": 1.0553395748138428,
"max": 1.376373291015625
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 217,
"p50": 6074359808.0,
"p95": 21595959296.0,
"mean": 9760589663.557604,
"min": 5234180096.0,
"max": 22246486016.0
},
"host_used_bytes": {
"n": 217,
"p50": 61699149824.0,
"p95": 61879255040.0,
"mean": 58012919968.4424,
"min": 45527023616.0,
"max": 62539329536.0
},
"host_cpu_percent": {
"n": 217,
"p50": 20.1,
"p95": 38.1,
"mean": 21.237327188940093,
"min": 10.0,
"max": 60.9
},
"gpu_memory_mib": {
"n": 217,
"p50": 23718.0,
"p95": 23730.0,
"mean": 18638.00921658986,
"min": 1618.0,
"max": 23732.0
},
"gpu_temperature_c": {
"n": 217,
"p50": 70.0,
"p95": 70.0,
"mean": 66.70506912442396,
"min": 56.0,
"max": 71.0
},
"gpu_power_w": {
"n": 217,
"p50": 76.3,
"p95": 81.11,
"mean": 62.00543778801843,
"min": 14.36,
"max": 88.57
},
"gpu_sm_clock_mhz": {
"n": 217,
"p50": 2730.0,
"p95": 2760.0,
"mean": 2161.5622119815666,
"min": 300.0,
"max": 2775.0
},
"gpu_mem_clock_mhz": {
"n": 217,
"p50": 14001.0,
"p95": 14001.0,
"mean": 11885.594470046082,
"min": 405.0,
"max": 14001.0
},
"gpu_utilization_percent": {
"n": 217,
"p50": 100.0,
"p95": 100.0,
"mean": 74.49308755760369,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 14525.52824447155,
"energy_observed_duration_s": 233.3380994796753,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 0,
"examples": []
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": null,
"host_launch_to_benchmark_report_start_s": 9.562777519226074,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage1_replica0 pid=1469796) INFO 09-22 17:32:17 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 2.768086 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=1469793) INFO 09-22 17:32:20 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 6.020106 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=1469796) INFO 09-22 17:35:10 [core.py:368] init engine (profile, create kv cache, warmup model) took 172.76 s",
"INFO 09-22 17:35:10 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed",
"(StageEngineCoreProc_stage0_replica0 pid=1469793) INFO 09-22 17:35:12 [monitor.py:53] torch.compile took 0.81 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=1469793) INFO 09-22 17:35:19 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=1469793) INFO 09-22 17:35:21 [core.py:361] init engine (profile, create kv cache, warmup model) took 180.06 s (compilation: 0.81 s)",
"INFO 09-22 17:35:21 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
},
"tts-cuda-wsl-trace": {
"whole_run": {
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-trace-host\\telemetry.jsonl",
"samples": 956,
"span_s": 1027.0037059783936,
"requested_window": {
"start": null,
"stop": null
},
"sampling_gap_s": {
"n": 955,
"p50": 1.0610792636871338,
"p95": 1.0773975849151611,
"mean": 1.0753965507627157,
"min": 1.0489082336425781,
"max": 3.0644443035125732
},
"gaps_over_5s": 0,
"metrics": {
"host_available_bytes": {
"n": 956,
"p50": 6321467392.0,
"p95": 15841251328.0,
"mean": 7510554846.794979,
"min": 212992.0,
"max": 24050614272.0
},
"host_used_bytes": {
"n": 956,
"p50": 61451980800.0,
"p95": 61780103168.0,
"mean": 60262954785.205025,
"min": 43722895360.0,
"max": 67773296640.0
},
"host_cpu_percent": {
"n": 956,
"p50": 11.4,
"p95": 22.9,
"mean": 13.044351464435145,
"min": 4.7,
"max": 57.9
},
"gpu_memory_mib": {
"n": 956,
"p50": 23932.0,
"p95": 23946.0,
"mean": 22090.27092050209,
"min": 1284.0,
"max": 23953.0
},
"gpu_temperature_c": {
"n": 956,
"p50": 58.0,
"p95": 59.0,
"mean": 58.22489539748954,
"min": 54.0,
"max": 60.0
},
"gpu_power_w": {
"n": 956,
"p50": 32.95,
"p95": 33.65,
"mean": 31.771412133891214,
"min": 12.99,
"max": 41.72
},
"gpu_sm_clock_mhz": {
"n": 956,
"p50": 1680.0,
"p95": 1702.0,
"mean": 1530.9623430962342,
"min": 180.0,
"max": 1950.0
},
"gpu_mem_clock_mhz": {
"n": 956,
"p50": 810.0,
"p95": 810.0,
"mean": 788.3943514644351,
"min": 405.0,
"max": 810.0
},
"gpu_utilization_percent": {
"n": 956,
"p50": 100.0,
"p95": 100.0,
"mean": 92.98430962343096,
"min": 0.0,
"max": 100.0
}
},
"ac_states": [
1
],
"whole_gpu_energy_j_over_observed_intervals": 32617.501377123594,
"energy_observed_duration_s": 1027.0037059783936,
"energy_scope": "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
},
"lifecycle_log_observations": {
"scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
"markers": {
"Dropping output for unknown req": {
"count": 0,
"examples": []
},
"force killing remaining process ": {
"count": 1,
"examples": [
"WARNING 09-22 18:57:03 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 1564626"
]
}
}
},
"startup": {
"cache_condition": "See source report",
"constructor_s": 937.4952322810423,
"host_launch_to_benchmark_report_start_s": 8.986722946166992,
"host_launch_to_first_request_s": null,
"host_launch_to_first_output_s": null,
"inclusive_startup_scope": "Host launch through first warmup submission/output includes imports, artifact hashing, plan construction and engine startup; uses recorded host/engine wall timestamps. Constructor timing alone excludes preparation before the constructor.",
"pre_constructor_scope": "Host launch to report creation includes Python/import/preparation work; not isolated OS process creation.",
"first_warmup": null,
"runtime_loading_compilation_evidence": [
"(StageEngineCoreProc_stage0_replica0 pid=1564626) INFO 09-22 18:40:32 [gpu_model_runner.py:5471] Model loading took 1.94 GiB memory and 3.105604 seconds",
"(StageEngineCoreProc_stage1_replica0 pid=1564629) INFO 09-22 18:40:33 [gpu_model_runner.py:5471] Model loading took 0.54 GiB memory and 4.036418 seconds",
"(StageEngineCoreProc_stage0_replica0 pid=1564626) INFO 09-22 18:40:34 [monitor.py:53] torch.compile took 0.19 s in total",
"(StageEngineCoreProc_stage0_replica0 pid=1564626) INFO 09-22 18:41:00 [gpu_ar_model_runner.py:632] Captured talker_mtp graphs for 4 sizes",
"(StageEngineCoreProc_stage0_replica0 pid=1564626) INFO 09-22 18:41:01 [core.py:361] init engine (profile, create kv cache, warmup model) took 28.82 s (compilation: 0.19 s)",
"INFO 09-22 18:41:02 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed",
"(StageEngineCoreProc_stage1_replica0 pid=1564629) INFO 09-22 18:55:48 [core.py:368] init engine (profile, create kv cache, warmup model) took 914.43 s",
"INFO 09-22 18:55:48 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed"
],
"scope": "Runtime-reported component lines retain original boundaries; do not sum parallel-stage times into startup latency."
}
}
},
"traces": [
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-trace\\traces\\1790073491843072609-rank-0.1790073527497371957.pt.trace.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 99,
"duration_sum_us": 5727378.5888671875,
"busy_union_us": 5727378.5888671875
},
"cpu_op": {
"events": 542460,
"duration_sum_us": 5381162.646484375,
"busy_union_us": 4374056.2802734375
},
"Trace": {
"events": 1,
"duration_sum_us": 5936096.849609375,
"busy_union_us": 5936096.849609375
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 5936096.85,
"duration_us": {
"n": 1,
"p50": 5936096.85,
"p95": 5936096.85,
"mean": 5936096.85,
"min": 5936096.85,
"max": 5936096.85
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 5495932.948,
"duration_us": {
"n": 93,
"p50": 59215.961,
"p95": 66924.524,
"mean": 59096.053204301075,
"min": 46803.984,
"max": 70379.831
}
},
{
"category": "cpu_op",
"name": "_C::onednn_scaled_mm",
"duration_sum_us": 2314593.276,
"duration_us": {
"n": 10752,
"p50": 188.999,
"p95": 455.926,
"mean": 215.2709520089286,
"min": 61.297,
"max": 909.159
}
},
{
"category": "cpu_op",
"name": "_C::onednn_mm",
"duration_sum_us": 683895.614,
"duration_us": {
"n": 96,
"p50": 7085.119,
"p95": 7775.247,
"mean": 7123.912645833333,
"min": 6435.122,
"max": 7938.207
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 235340.251,
"duration_us": {
"n": 52620,
"p50": 2.074,
"p95": 10.049,
"mean": 4.472448707715698,
"min": 0.07,
"max": 585.916
}
},
{
"category": "user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 230681.874,
"duration_us": {
"n": 3,
"p50": 74541.32,
"p95": 81792.19,
"mean": 76893.958,
"min": 74348.364,
"max": 81792.19
}
},
{
"category": "cpu_op",
"name": "vllm::unified_kv_cache_update",
"duration_sum_us": 212245.381,
"duration_us": {
"n": 2688,
"p50": 69.612,
"p95": 146.629,
"mean": 78.96033519345238,
"min": 28.535,
"max": 412.012
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 204785.018,
"duration_us": {
"n": 41196,
"p50": 2.174,
"p95": 9.949,
"mean": 4.970992766287989,
"min": 0.641,
"max": 581.578
}
},
{
"category": "cpu_op",
"name": "vllm::unified_attention_with_output",
"duration_sum_us": 171424.494,
"duration_us": {
"n": 2688,
"p50": 56.357,
"p95": 107.304,
"mean": 63.77399330357143,
"min": 27.833,
"max": 443.228
}
},
{
"category": "cpu_op",
"name": "aten::gelu",
"duration_sum_us": 164319.428,
"duration_us": {
"n": 2688,
"p50": 53.051,
"p95": 107.234,
"mean": 61.13073958333334,
"min": 20.67,
"max": 406.832
}
},
{
"category": "cpu_op",
"name": "vllm::spark_head_gate",
"duration_sum_us": 143424.632,
"duration_us": {
"n": 2688,
"p50": 43.332,
"p95": 109.609,
"mean": 53.35737797619048,
"min": 19.417,
"max": 438.383
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 118827.177,
"duration_us": {
"n": 42636,
"p50": 1.213,
"p95": 6.091,
"mean": 2.7870151280607938,
"min": 0.04,
"max": 444.913
}
},
{
"category": "cpu_op",
"name": "aten::mean",
"duration_sum_us": 110941.426,
"duration_us": {
"n": 5472,
"p50": 14.507,
"p95": 43.603,
"mean": 20.27438340643275,
"min": 4.509,
"max": 456.582
}
},
{
"category": "cpu_op",
"name": "_C::dynamic_scaled_int8_quant",
"duration_sum_us": 103954.003,
"duration_us": {
"n": 10752,
"p50": 4.889,
"p95": 22.913,
"mean": 9.668341052827381,
"min": 1.894,
"max": 584.931
}
},
{
"category": "cpu_op",
"name": "aten::div_",
"duration_sum_us": 96764.749,
"duration_us": {
"n": 10944,
"p50": 6.422,
"p95": 17.413,
"mean": 8.841808205409356,
"min": 2.184,
"max": 443.217
}
},
{
"category": "cpu_op",
"name": "aten::mul",
"duration_sum_us": 63436.993,
"duration_us": {
"n": 18924,
"p50": 2.014,
"p95": 8.716,
"mean": 3.3521978968505604,
"min": 0.541,
"max": 297.024
}
},
{
"category": "cpu_op",
"name": "aten::sum",
"duration_sum_us": 61086.44,
"duration_us": {
"n": 8076,
"p50": 5.841,
"p95": 15.941,
"mean": 7.563947498761763,
"min": 0.881,
"max": 257.298
}
},
{
"category": "cpu_op",
"name": "aten::chunk",
"duration_sum_us": 58636.218,
"duration_us": {
"n": 5376,
"p50": 7.905,
"p95": 23.164,
"mean": 10.907034598214286,
"min": 1.623,
"max": 269.903
}
},
{
"category": "cpu_op",
"name": "_C::cpu_attention_with_kv_cache",
"duration_sum_us": 55159.251,
"duration_us": {
"n": 2688,
"p50": 17.954,
"p95": 33.714,
"mean": 20.5205546875,
"min": 8.166,
"max": 364.639
}
},
{
"category": "cpu_op",
"name": "aten::add",
"duration_sum_us": 50621.96,
"duration_us": {
"n": 11136,
"p50": 3.046,
"p95": 9.158,
"mean": 4.54579382183908,
"min": 0.411,
"max": 295.341
}
},
{
"category": "cpu_op",
"name": "aten::slice",
"duration_sum_us": 45857.671,
"duration_us": {
"n": 27570,
"p50": 0.712,
"p95": 4.328,
"mean": 1.6633177729416033,
"min": 0.23,
"max": 244.144
}
},
{
"category": "cpu_op",
"name": "aten::empty_strided",
"duration_sum_us": 42035.924,
"duration_us": {
"n": 62796,
"p50": 0.361,
"p95": 1.793,
"mean": 0.6694044843620612,
"min": 0.11,
"max": 274.916
}
},
{
"category": "cpu_op",
"name": "aten::split",
"duration_sum_us": 41479.545,
"duration_us": {
"n": 5376,
"p50": 5.58,
"p95": 14.758,
"mean": 7.715689174107142,
"min": 1.272,
"max": 265.865
}
},
{
"category": "cpu_op",
"name": "_C::cpu_attn_reshape_and_cache",
"duration_sum_us": 40454.911,
"duration_us": {
"n": 2688,
"p50": 9.448,
"p95": 38.313,
"mean": 15.050190104166667,
"min": 4.459,
"max": 287.014
}
},
{
"category": "cpu_op",
"name": "aten::view",
"duration_sum_us": 34457.299,
"duration_us": {
"n": 35424,
"p50": 0.4,
"p95": 3.928,
"mean": 0.9727105634598012,
"min": 0.06,
"max": 120.268
}
},
{
"category": "cpu_op",
"name": "aten::empty_like",
"duration_sum_us": 34197.286,
"duration_us": {
"n": 21504,
"p50": 0.872,
"p95": 3.337,
"mean": 1.5902755766369048,
"min": 0.26,
"max": 358.541
}
},
{
"category": "cpu_op",
"name": "aten::pow",
"duration_sum_us": 32479.801,
"duration_us": {
"n": 5472,
"p50": 4.158,
"p95": 11.772,
"mean": 5.935636147660818,
"min": 1.193,
"max": 370.531
}
},
{
"category": "cpu_op",
"name": "aten::narrow",
"duration_sum_us": 31538.473,
"duration_us": {
"n": 10752,
"p50": 1.142,
"p95": 7.424,
"mean": 2.9332657180059525,
"min": 0.421,
"max": 260.084
}
},
{
"category": "cpu_op",
"name": "_C::rotary_embedding",
"duration_sum_us": 24278.963,
"duration_us": {
"n": 2688,
"p50": 5.831,
"p95": 20.729,
"mean": 9.032352306547619,
"min": 3.277,
"max": 277.607
}
},
{
"category": "cpu_op",
"name": "aten::as_strided",
"duration_sum_us": 23314.983,
"duration_us": {
"n": 44286,
"p50": 0.27,
"p95": 1.653,
"mean": 0.5264639615228289,
"min": 0.04,
"max": 106.322
}
},
{
"category": "cpu_op",
"name": "aten::empty",
"duration_sum_us": 20411.026,
"duration_us": {
"n": 27456,
"p50": 0.451,
"p95": 1.844,
"mean": 0.7434085810023311,
"min": 0.11,
"max": 241.899
}
},
{
"category": "cpu_op",
"name": "aten::item",
"duration_sum_us": 19040.216,
"duration_us": {
"n": 10944,
"p50": 1.112,
"p95": 3.236,
"mean": 1.7397858187134503,
"min": 0.24,
"max": 371.275
}
},
{
"category": "cpu_op",
"name": "aten::split_with_sizes",
"duration_sum_us": 13733.253,
"duration_us": {
"n": 2688,
"p50": 3.527,
"p95": 8.756,
"mean": 5.109097098214286,
"min": 0.992,
"max": 248.212
}
},
{
"category": "cpu_op",
"name": "aten::fill_",
"duration_sum_us": 12823.427,
"duration_us": {
"n": 9228,
"p50": 0.922,
"p95": 3.708,
"mean": 1.3896214781100997,
"min": 0.06,
"max": 109.918
}
},
{
"category": "cpu_op",
"name": "aten::argmax",
"duration_sum_us": 11892.041,
"duration_us": {
"n": 96,
"p50": 121.671,
"p95": 211.101,
"mean": 123.87542708333332,
"min": 81.455,
"max": 235.247
}
},
{
"category": "cpu_op",
"name": "aten::rsqrt",
"duration_sum_us": 10755.292,
"duration_us": {
"n": 5472,
"p50": 1.513,
"p95": 4.198,
"mean": 1.9655138888888888,
"min": 0.38,
"max": 116.672
}
},
{
"category": "cpu_op",
"name": "aten::linear",
"duration_sum_us": 9396.005,
"duration_us": {
"n": 84,
"p50": 84.68,
"p95": 304.238,
"mean": 111.85720238095237,
"min": 50.927,
"max": 360.655
}
},
{
"category": "cpu_op",
"name": "aten::unflatten",
"duration_sum_us": 9218.381,
"duration_us": {
"n": 2688,
"p50": 2.655,
"p95": 4.99,
"mean": 3.4294572172619047,
"min": 1.002,
"max": 273.109
}
},
{
"category": "cpu_op",
"name": "aten::unsqueeze",
"duration_sum_us": 9137.108,
"duration_us": {
"n": 5388,
"p50": 1.142,
"p95": 2.866,
"mean": 1.6958255382331107,
"min": 0.32,
"max": 229.095
}
},
{
"category": "cpu_op",
"name": "aten::matmul",
"duration_sum_us": 8670.965,
"duration_us": {
"n": 84,
"p50": 77.207,
"p95": 262.078,
"mean": 103.22577380952382,
"min": 47.17,
"max": 347.991
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cpu-wsl-trace-full\\traces\\1790060202312950501-rank-0.1790060302118768569.pt.trace.json.gz",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 387,
"duration_sum_us": 25657999.313476562,
"busy_union_us": 25657999.313476562
},
"cpu_op": {
"events": 2170236,
"duration_sum_us": 25267840.64453125,
"busy_union_us": 19370283.249023438
},
"Trace": {
"events": 1,
"duration_sum_us": 26502152.181640625,
"busy_union_us": 26502152.181640625
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 26502152.182,
"duration_us": {
"n": 1,
"p50": 26502152.182,
"p95": 26502152.182,
"mean": 26502152.182,
"min": 26502152.182,
"max": 26502152.182
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 25420971.336,
"duration_us": {
"n": 381,
"p50": 65787.316,
"p95": 78369.913,
"mean": 66721.70954330708,
"min": 53843.793,
"max": 84857.183
}
},
{
"category": "cpu_op",
"name": "_C::onednn_scaled_mm",
"duration_sum_us": 9436726.179,
"duration_us": {
"n": 43008,
"p50": 191.854,
"p95": 469.672,
"mean": 219.41792640904018,
"min": 61.246,
"max": 1466.791
}
},
{
"category": "cpu_op",
"name": "_C::onednn_mm",
"duration_sum_us": 2715319.96,
"duration_us": {
"n": 384,
"p50": 7033.717,
"p95": 7641.191,
"mean": 7071.145729166667,
"min": 6435.131,
"max": 8973.952
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 1479147.211,
"duration_us": {
"n": 210732,
"p50": 3.286,
"p95": 15.99,
"mean": 7.0190915997570364,
"min": 0.09,
"max": 1426.028
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 1244996.039,
"duration_us": {
"n": 165036,
"p50": 3.386,
"p95": 15.439,
"mean": 7.543784622748976,
"min": 0.872,
"max": 1425.156
}
},
{
"category": "cpu_op",
"name": "vllm::unified_kv_cache_update",
"duration_sum_us": 1011477.538,
"duration_us": {
"n": 10752,
"p50": 80.794,
"p95": 198.026,
"mean": 94.07343173363094,
"min": 32.522,
"max": 1682.385
}
},
{
"category": "cpu_op",
"name": "vllm::unified_attention_with_output",
"duration_sum_us": 952406.203,
"duration_us": {
"n": 10752,
"p50": 71.747,
"p95": 195.631,
"mean": 88.57944596354166,
"min": 33.033,
"max": 1254.743
}
},
{
"category": "cpu_op",
"name": "vllm::spark_head_gate",
"duration_sum_us": 740232.488,
"duration_us": {
"n": 10752,
"p50": 52.56,
"p95": 164.382,
"mean": 68.84602752976191,
"min": 21.721,
"max": 1319.686
}
},
{
"category": "cpu_op",
"name": "aten::gelu",
"duration_sum_us": 700408.052,
"duration_us": {
"n": 10752,
"p50": 57.789,
"p95": 112.023,
"mean": 65.14211793154762,
"min": 19.657,
"max": 855.895
}
},
{
"category": "cpu_op",
"name": "aten::mean",
"duration_sum_us": 642115.725,
"duration_us": {
"n": 21888,
"p50": 20.339,
"p95": 70.243,
"mean": 29.336427494517544,
"min": 6.362,
"max": 1219.306
}
},
{
"category": "cpu_op",
"name": "aten::div_",
"duration_sum_us": 601245.17,
"duration_us": {
"n": 43776,
"p50": 9.237,
"p95": 27.572,
"mean": 13.734584475511697,
"min": 2.826,
"max": 1208.645
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 506640.196,
"duration_us": {
"n": 170796,
"p50": 1.403,
"p95": 5.952,
"mean": 2.9663469636291246,
"min": 0.07,
"max": 1094.158
}
},
{
"category": "cpu_op",
"name": "aten::chunk",
"duration_sum_us": 419007.945,
"duration_us": {
"n": 21504,
"p50": 12.564,
"p95": 55.084,
"mean": 19.485116489955356,
"min": 3.006,
"max": 1142.741
}
},
{
"category": "cpu_op",
"name": "_C::dynamic_scaled_int8_quant",
"duration_sum_us": 386736.611,
"duration_us": {
"n": 43008,
"p50": 4.78,
"p95": 21.601,
"mean": 8.992201706659225,
"min": 1.953,
"max": 572.828
}
},
{
"category": "cpu_op",
"name": "aten::slice",
"duration_sum_us": 359223.62,
"duration_us": {
"n": 110226,
"p50": 1.253,
"p95": 8.336,
"mean": 3.258973563406093,
"min": 0.381,
"max": 1117.792
}
},
{
"category": "cpu_op",
"name": "aten::empty_strided",
"duration_sum_us": 332838.286,
"duration_us": {
"n": 251436,
"p50": 0.521,
"p95": 2.996,
"mean": 1.3237495267185289,
"min": 0.15,
"max": 803.165
}
},
{
"category": "cpu_op",
"name": "aten::split",
"duration_sum_us": 328926.75,
"duration_us": {
"n": 21504,
"p50": 9.398,
"p95": 39.746,
"mean": 15.296072823660714,
"min": 2.465,
"max": 1123.083
}
},
{
"category": "cpu_op",
"name": "aten::mul",
"duration_sum_us": 304211.119,
"duration_us": {
"n": 75948,
"p50": 2.495,
"p95": 9.588,
"mean": 4.005518499499658,
"min": 0.631,
"max": 1183.558
}
},
{
"category": "cpu_op",
"name": "aten::sum",
"duration_sum_us": 300495.749,
"duration_us": {
"n": 32556,
"p50": 6.823,
"p95": 17.493,
"mean": 9.230118841381005,
"min": 1.273,
"max": 1210.66
}
},
{
"category": "cpu_op",
"name": "aten::add",
"duration_sum_us": 296251.915,
"duration_us": {
"n": 44544,
"p50": 4.018,
"p95": 13.546,
"mean": 6.650770361889367,
"min": 0.531,
"max": 1072.617
}
},
{
"category": "cpu_op",
"name": "_C::cpu_attention_with_kv_cache",
"duration_sum_us": 285393.383,
"duration_us": {
"n": 10752,
"p50": 23.084,
"p95": 44.304,
"mean": 26.543283389136903,
"min": 8.766,
"max": 966.676
}
},
{
"category": "cpu_op",
"name": "aten::empty_like",
"duration_sum_us": 273730.191,
"duration_us": {
"n": 86016,
"p50": 1.393,
"p95": 6.752,
"mean": 3.1823171386718747,
"min": 0.411,
"max": 1122.582
}
},
{
"category": "cpu_op",
"name": "aten::narrow",
"duration_sum_us": 253346.735,
"duration_us": {
"n": 43008,
"p50": 2.194,
"p95": 12.794,
"mean": 5.89068859281994,
"min": 0.791,
"max": 1119.155
}
},
{
"category": "user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 236180.194,
"duration_us": {
"n": 3,
"p50": 79055.223,
"p95": 82096.078,
"mean": 78726.73133333333,
"min": 75028.893,
"max": 82096.078
}
},
{
"category": "cpu_op",
"name": "aten::view",
"duration_sum_us": 209848.78,
"duration_us": {
"n": 141696,
"p50": 0.561,
"p95": 5.11,
"mean": 1.4809788561427282,
"min": 0.1,
"max": 788.377
}
},
{
"category": "cpu_op",
"name": "aten::pow",
"duration_sum_us": 180900.408,
"duration_us": {
"n": 21888,
"p50": 5.19,
"p95": 14.888,
"mean": 8.264821271929824,
"min": 1.523,
"max": 542.26
}
},
{
"category": "cpu_op",
"name": "aten::empty",
"duration_sum_us": 165310.618,
"duration_us": {
"n": 109824,
"p50": 0.681,
"p95": 3.106,
"mean": 1.5052321714743588,
"min": 0.15,
"max": 955.234
}
},
{
"category": "cpu_op",
"name": "aten::as_strided",
"duration_sum_us": 154764.813,
"duration_us": {
"n": 177342,
"p50": 0.331,
"p95": 2.154,
"mean": 0.8726912575701187,
"min": 0.05,
"max": 767.577
}
},
{
"category": "cpu_op",
"name": "_C::cpu_attn_reshape_and_cache",
"duration_sum_us": 146209.097,
"duration_us": {
"n": 10752,
"p50": 9.498,
"p95": 27.211,
"mean": 13.598316313244048,
"min": 4.308,
"max": 1598.014
}
},
{
"category": "cpu_op",
"name": "aten::split_with_sizes",
"duration_sum_us": 99767.459,
"duration_us": {
"n": 10752,
"p50": 5.21,
"p95": 18.214,
"mean": 9.278967540922618,
"min": 1.844,
"max": 1013.314
}
},
{
"category": "cpu_op",
"name": "_C::rotary_embedding",
"duration_sum_us": 94416.149,
"duration_us": {
"n": 10752,
"p50": 5.671,
"p95": 20.068,
"mean": 8.781263857886906,
"min": 2.865,
"max": 486.975
}
},
{
"category": "cpu_op",
"name": "aten::item",
"duration_sum_us": 85718.681,
"duration_us": {
"n": 43776,
"p50": 1.212,
"p95": 3.566,
"mean": 1.958120454130117,
"min": 0.22,
"max": 538.643
}
},
{
"category": "cpu_op",
"name": "aten::unsqueeze",
"duration_sum_us": 73909.101,
"duration_us": {
"n": 21804,
"p50": 2.014,
"p95": 5.981,
"mean": 3.3897037699504677,
"min": 0.601,
"max": 769.622
}
},
{
"category": "cpu_op",
"name": "aten::rsqrt",
"duration_sum_us": 61391.475,
"duration_us": {
"n": 21888,
"p50": 1.873,
"p95": 5.13,
"mean": 2.804800575657895,
"min": 0.451,
"max": 619.296
}
},
{
"category": "cpu_op",
"name": "aten::unflatten",
"duration_sum_us": 55410.19,
"duration_us": {
"n": 10752,
"p50": 4.098,
"p95": 8.205,
"mean": 5.153477492559524,
"min": 1.373,
"max": 235.617
}
},
{
"category": "cpu_op",
"name": "aten::fill_",
"duration_sum_us": 52097.08,
"duration_us": {
"n": 37164,
"p50": 0.842,
"p95": 3.466,
"mean": 1.4018157356581638,
"min": 0.051,
"max": 138.754
}
},
{
"category": "cpu_op",
"name": "aten::argmax",
"duration_sum_us": 51511.468,
"duration_us": {
"n": 384,
"p50": 129.376,
"p95": 208.777,
"mean": 134.14444791666668,
"min": 82.236,
"max": 392.68
}
},
{
"category": "cpu_op",
"name": "aten::flatten",
"duration_sum_us": 40282.293,
"duration_us": {
"n": 11136,
"p50": 2.314,
"p95": 8.496,
"mean": 3.617303609913793,
"min": 0.812,
"max": 697.024
}
},
{
"category": "cpu_op",
"name": "aten::sigmoid",
"duration_sum_us": 39351.278,
"duration_us": {
"n": 10752,
"p50": 2.956,
"p95": 6.633,
"mean": 3.659903087797619,
"min": 1.142,
"max": 123.214
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-windows-trace\\traces\\20260922-171601_e2e-diagnostic\\trace_rank0.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 102,
"duration_sum_us": 5182923.0029296875,
"busy_union_us": 5182923.0029296875
},
"cpu_op": {
"events": 388314,
"duration_sum_us": 7239074.1357421875,
"busy_union_us": 3611484.796875
},
"overhead": {
"events": 14,
"duration_sum_us": 6320.8994140625,
"busy_union_us": 6320.8994140625
},
"kernel": {
"events": 54084,
"duration_sum_us": 1219158.0869140625,
"busy_union_us": 1219158.0869140625
},
"cuda_runtime": {
"events": 69616,
"duration_sum_us": 1136353.134765625,
"busy_union_us": 1136353.134765625
},
"gpu_memcpy": {
"events": 2181,
"duration_sum_us": 2501.189453125,
"busy_union_us": 2501.189453125
},
"gpu_memset": {
"events": 204,
"duration_sum_us": 67.8369140625,
"busy_union_us": 67.8369140625
},
"cuda_driver": {
"events": 924,
"duration_sum_us": 15317.3125,
"busy_union_us": 15317.3125
},
"gpu_user_annotation": {
"events": 96,
"duration_sum_us": 5224279.03125,
"busy_union_us": 5224279.03125
},
"Trace": {
"events": 1,
"duration_sum_us": 5413389.599609375,
"busy_union_us": 5413389.599609375
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 5413389.6,
"duration_us": {
"n": 1,
"p50": 5413389.6,
"p95": 5413389.6,
"mean": 5413389.6,
"min": 5413389.6,
"max": 5413389.6
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 5049678.437,
"duration_us": {
"n": 93,
"p50": 53276.41,
"p95": 67162.558,
"mean": 54297.61760215054,
"min": 41361.873,
"max": 82459.674
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 5002969.3,
"duration_us": {
"n": 93,
"p50": 52943.0,
"p95": 66714.5,
"mean": 53795.3688172043,
"min": 40975.5,
"max": 82146.2
}
},
{
"category": "cuda_runtime",
"name": "cudaLaunchKernel",
"duration_sum_us": 1022939.4,
"duration_us": {
"n": 53160,
"p50": 13.5,
"p95": 48.1,
"mean": 19.24265237020316,
"min": 3.6,
"max": 2082.9
}
},
{
"category": "cpu_op",
"name": "vllm::unified_attention_with_output",
"duration_sum_us": 1011700.1,
"duration_us": {
"n": 3456,
"p50": 255.6,
"p95": 600.1,
"mean": 292.7372974537037,
"min": 96.7,
"max": 1602.7
}
},
{
"category": "cpu_op",
"name": "aten::linear",
"duration_sum_us": 986526.1,
"duration_us": {
"n": 17376,
"p50": 42.2,
"p95": 134.0,
"mean": 56.7752129373849,
"min": 15.5,
"max": 821.4
}
},
{
"category": "kernel",
"name": "_ZN8internal5gemvx6kernelIii13__nv_bfloat16S2_S2_fLb0ELb1ELb1ELb0ELi6ELb0E18cublasGemvParamsExIi30cublasGemvTensorStridedBatchedIKS2_ES6_S4_IS2_EfEEENSt9enable_ifIXntT5_EvE4typeET11_",
"duration_sum_us": 843376.31,
"duration_us": {
"n": 10140,
"p50": 40.257,
"p95": 189.316,
"mean": 83.17320611439843,
"min": 26.528,
"max": 3406.318
}
},
{
"category": "cpu_op",
"name": "aten::matmul",
"duration_sum_us": 790111.6,
"duration_us": {
"n": 17376,
"p50": 33.8,
"p95": 108.5,
"mean": 45.47143186003683,
"min": 11.7,
"max": 707.8
}
},
{
"category": "cpu_op",
"name": "aten::mm",
"duration_sum_us": 746332.0,
"duration_us": {
"n": 17376,
"p50": 32.0,
"p95": 102.9,
"mean": 42.951887661141804,
"min": 10.9,
"max": 705.3
}
},
{
"category": "cpu_op",
"name": "_vllm_fa2_C::varlen_fwd",
"duration_sum_us": 470222.3,
"duration_us": {
"n": 3456,
"p50": 98.9,
"p95": 328.7,
"mean": 136.05969328703702,
"min": 32.0,
"max": 865.0
}
},
{
"category": "cpu_op",
"name": "vllm::unified_kv_cache_update",
"duration_sum_us": 415035.7,
"duration_us": {
"n": 3456,
"p50": 102.3,
"p95": 241.6,
"mean": 120.09134837962964,
"min": 48.9,
"max": 1139.8
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 335008.1,
"duration_us": {
"n": 7968,
"p50": 32.5,
"p95": 98.2,
"mean": 42.04418925702811,
"min": 0.3,
"max": 695.2
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 317818.0,
"duration_us": {
"n": 9765,
"p50": 23.5,
"p95": 77.3,
"mean": 32.54664618535586,
"min": 1.4,
"max": 681.4
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 314510.8,
"duration_us": {
"n": 7488,
"p50": 32.1,
"p95": 94.7,
"mean": 42.001976495726495,
"min": 10.7,
"max": 693.1
}
},
{
"category": "kernel",
"name": "_ZN8internal5gemvx6kernelIii13__nv_bfloat16S2_S2_fLb0ELb1ELb1ELb0ELi7ELb0E18cublasGemvParamsExIi30cublasGemvTensorStridedBatchedIKS2_ES6_S4_IS2_EfEEENSt9enable_ifIXntT5_EvE4typeET11_",
"duration_sum_us": 261555.237,
"duration_us": {
"n": 6696,
"p50": 63.553,
"p95": 96.898,
"mean": 39.06141532258064,
"min": 2.559,
"max": 2607.611
}
},
{
"category": "cpu_op",
"name": "_C::fused_add_rms_norm",
"duration_sum_us": 185624.5,
"duration_us": {
"n": 6912,
"p50": 18.7,
"p95": 66.9,
"mean": 26.855396412037038,
"min": 6.3,
"max": 578.3
}
},
{
"category": "user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 177660.8,
"duration_us": {
"n": 3,
"p50": 59561.9,
"p95": 59598.1,
"mean": 59220.26666666666,
"min": 58500.8,
"max": 59598.1
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 174600.595,
"duration_us": {
"n": 3,
"p50": 58514.488,
"p95": 59616.594,
"mean": 58200.198333333334,
"min": 56469.513,
"max": 59616.594
}
},
{
"category": "cpu_op",
"name": "_C_cache_ops::reshape_and_cache_flash",
"duration_sum_us": 131588.9,
"duration_us": {
"n": 3456,
"p50": 31.7,
"p95": 81.0,
"mean": 38.075491898148144,
"min": 11.3,
"max": 440.2
}
},
{
"category": "cpu_op",
"name": "aten::split",
"duration_sum_us": 123684.5,
"duration_us": {
"n": 6912,
"p50": 13.4,
"p95": 29.2,
"mean": 17.894169560185187,
"min": 5.2,
"max": 523.1
}
},
{
"category": "cpu_op",
"name": "aten::mul",
"duration_sum_us": 118623.1,
"duration_us": {
"n": 3456,
"p50": 28.8,
"p95": 70.7,
"mean": 34.323813657407406,
"min": 10.9,
"max": 639.7
}
},
{
"category": "cpu_op",
"name": "aten::t",
"duration_sum_us": 118248.0,
"duration_us": {
"n": 17376,
"p50": 4.9,
"p95": 12.4,
"mean": 6.80524861878453,
"min": 1.9,
"max": 511.2
}
},
{
"category": "cpu_op",
"name": "aten::transpose",
"duration_sum_us": 104528.0,
"duration_us": {
"n": 26799,
"p50": 2.8,
"p95": 7.6,
"mean": 3.9004440464196426,
"min": 1.1,
"max": 614.0
}
},
{
"category": "cpu_op",
"name": "aten::sigmoid",
"duration_sum_us": 96850.5,
"duration_us": {
"n": 3456,
"p50": 22.9,
"p95": 55.8,
"mean": 28.02387152777778,
"min": 8.5,
"max": 525.6
}
},
{
"category": "cpu_op",
"name": "_C::rotary_embedding",
"duration_sum_us": 95075.3,
"duration_us": {
"n": 3456,
"p50": 20.8,
"p95": 64.8,
"mean": 27.51021412037037,
"min": 9.3,
"max": 623.6
}
},
{
"category": "cpu_op",
"name": "aten::empty",
"duration_sum_us": 89090.2,
"duration_us": {
"n": 17472,
"p50": 3.9,
"p95": 11.2,
"mean": 5.099027014652014,
"min": 1.3,
"max": 404.0
}
},
{
"category": "cpu_op",
"name": "_C::gelu_and_mul",
"duration_sum_us": 85345.4,
"duration_us": {
"n": 3456,
"p50": 18.7,
"p95": 55.7,
"mean": 24.694849537037037,
"min": 6.8,
"max": 434.9
}
},
{
"category": "cpu_op",
"name": "aten::slice",
"duration_sum_us": 84464.9,
"duration_us": {
"n": 26709,
"p50": 2.2,
"p95": 4.9,
"mean": 3.162413418697817,
"min": 0.8,
"max": 593.1
}
},
{
"category": "cpu_op",
"name": "aten::empty_strided",
"duration_sum_us": 79396.5,
"duration_us": {
"n": 11877,
"p50": 5.3,
"p95": 15.2,
"mean": 6.684895175549381,
"min": 2.0,
"max": 322.9
}
},
{
"category": "cpu_op",
"name": "aten::narrow",
"duration_sum_us": 79100.2,
"duration_us": {
"n": 13824,
"p50": 4.0,
"p95": 10.4,
"mean": 5.721947337962963,
"min": 1.6,
"max": 506.7
}
},
{
"category": "cpu_op",
"name": "aten::new_empty",
"duration_sum_us": 76827.8,
"duration_us": {
"n": 10368,
"p50": 5.7,
"p95": 13.6,
"mean": 7.410088734567902,
"min": 2.2,
"max": 508.7
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 73375.0,
"duration_us": {
"n": 2181,
"p50": 31.8,
"p95": 71.9,
"mean": 33.64282439248051,
"min": 4.6,
"max": 414.5
}
},
{
"category": "cpu_op",
"name": "aten::as_strided",
"duration_sum_us": 61698.2,
"duration_us": {
"n": 78282,
"p50": 0.6,
"p95": 1.9,
"mean": 0.7881530875552489,
"min": 0.1,
"max": 199.8
}
},
{
"category": "cpu_op",
"name": "aten::empty_like",
"duration_sum_us": 50457.8,
"duration_us": {
"n": 4293,
"p50": 9.5,
"p95": 21.7,
"mean": 11.753505706964827,
"min": 4.1,
"max": 364.5
}
},
{
"category": "cpu_op",
"name": "aten::expand",
"duration_sum_us": 45511.4,
"duration_us": {
"n": 10371,
"p50": 3.3,
"p95": 7.2,
"mean": 4.388332851219747,
"min": 1.4,
"max": 396.2
}
},
{
"category": "cpu_op",
"name": "aten::view",
"duration_sum_us": 30690.5,
"duration_us": {
"n": 27927,
"p50": 0.6,
"p95": 3.9,
"mean": 1.0989544168725607,
"min": 0.1,
"max": 639.1
}
},
{
"category": "cpu_op",
"name": "aten::split_with_sizes",
"duration_sum_us": 29590.2,
"duration_us": {
"n": 3456,
"p50": 6.9,
"p95": 12.3,
"mean": 8.561979166666667,
"min": 2.9,
"max": 312.2
}
},
{
"category": "cpu_op",
"name": "aten::reshape",
"duration_sum_us": 26046.6,
"duration_us": {
"n": 4377,
"p50": 3.7,
"p95": 12.5,
"mean": 5.950788211103495,
"min": 1.2,
"max": 396.2
}
},
{
"category": "cpu_op",
"name": "aten::index",
"duration_sum_us": 21845.2,
"duration_us": {
"n": 192,
"p50": 93.9,
"p95": 239.8,
"mean": 113.77708333333334,
"min": 45.9,
"max": 632.6
}
},
{
"category": "cuda_runtime",
"name": "cudaFuncSetAttribute",
"duration_sum_us": 18844.0,
"duration_us": {
"n": 3456,
"p50": 4.8,
"p95": 10.1,
"mean": 5.452546296296297,
"min": 1.6,
"max": 63.7
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\spark-cuda-wsl-trace\\traces\\20260922-171224_e2e-diagnostic\\trace_rank0.json.gz",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 102,
"duration_sum_us": 2108171.9814453125,
"busy_union_us": 2108171.9814453125
},
"cpu_op": {
"events": 388314,
"duration_sum_us": 2892456.63671875,
"busy_union_us": 1447947.6484375
},
"overhead": {
"events": 14,
"duration_sum_us": 53227.46484375,
"busy_union_us": 53227.46484375
},
"cuda_runtime": {
"events": 69644,
"duration_sum_us": 773948.5751953125,
"busy_union_us": 773948.5751953125
},
"gpu_memcpy": {
"events": 2181,
"duration_sum_us": 2914.7314453125,
"busy_union_us": 2914.7314453125
},
"kernel": {
"events": 54084,
"duration_sum_us": 1941159.0244140625,
"busy_union_us": 1941159.0244140625
},
"cuda_driver": {
"events": 924,
"duration_sum_us": 9172.794921875,
"busy_union_us": 9172.794921875
},
"gpu_memset": {
"events": 204,
"duration_sum_us": 89.0556640625,
"busy_union_us": 89.0556640625
},
"gpu_user_annotation": {
"events": 96,
"duration_sum_us": 2407730.3759765625,
"busy_union_us": 2407730.3759765625
},
"Trace": {
"events": 1,
"duration_sum_us": 2445941.345703125,
"busy_union_us": 2445941.345703125
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 2445941.346,
"duration_us": {
"n": 1,
"p50": 2445941.346,
"p95": 2445941.346,
"mean": 2445941.346,
"min": 2445941.346,
"max": 2445941.346
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 2308408.933,
"duration_us": {
"n": 93,
"p50": 23689.315,
"p95": 34071.886,
"mean": 24821.60143010753,
"min": 17970.8,
"max": 59028.049
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 2012183.098,
"duration_us": {
"n": 93,
"p50": 21247.04,
"p95": 27493.991,
"mean": 21636.37739784946,
"min": 15591.598,
"max": 41813.503
}
},
{
"category": "kernel",
"name": "std::enable_if<!(false), void>::type internal::gemvx::kernel<int, int, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, float, false, true, true, false, 6, false, cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float> >(cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float>)",
"duration_sum_us": 1319045.228,
"duration_us": {
"n": 10140,
"p50": 54.529,
"p95": 212.774,
"mean": 130.0833558185404,
"min": 30.529,
"max": 5712.914
}
},
{
"category": "cuda_runtime",
"name": "cudaLaunchKernel",
"duration_sum_us": 498947.253,
"duration_us": {
"n": 52836,
"p50": 6.843,
"p95": 20.428,
"mean": 9.443319952305247,
"min": 3.947,
"max": 17110.162
}
},
{
"category": "kernel",
"name": "std::enable_if<!(false), void>::type internal::gemvx::kernel<int, int, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, float, false, true, true, false, 7, false, cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float> >(cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float>)",
"duration_sum_us": 413813.511,
"duration_us": {
"n": 6696,
"p50": 48.769,
"p95": 112.771,
"mean": 61.8001061827957,
"min": 3.104,
"max": 3771.68
}
},
{
"category": "cpu_op",
"name": "aten::linear",
"duration_sum_us": 390467.921,
"duration_us": {
"n": 17376,
"p50": 15.901,
"p95": 49.965,
"mean": 22.471680536372006,
"min": 10.621,
"max": 17199.241
}
},
{
"category": "cpu_op",
"name": "vllm::unified_attention_with_output",
"duration_sum_us": 356136.782,
"duration_us": {
"n": 3456,
"p50": 82.957,
"p95": 204.869,
"mean": 103.04883738425926,
"min": 48.653,
"max": 4194.665
}
},
{
"category": "cpu_op",
"name": "aten::matmul",
"duration_sum_us": 328175.829,
"duration_us": {
"n": 17376,
"p50": 13.335,
"p95": 43.021,
"mean": 18.886730490331495,
"min": 8.857,
"max": 17186.086
}
},
{
"category": "cpu_op",
"name": "aten::mm",
"duration_sum_us": 315288.947,
"duration_us": {
"n": 17376,
"p50": 12.815,
"p95": 41.839,
"mean": 18.145082124769797,
"min": 8.466,
"max": 17171.349
}
},
{
"category": "cuda_runtime",
"name": "cudaEventSynchronize",
"duration_sum_us": 214946.927,
"duration_us": {
"n": 198,
"p50": 4.188,
"p95": 7580.024,
"mean": 1085.5905404040404,
"min": 2.855,
"max": 41591.928
}
},
{
"category": "cpu_op",
"name": "_vllm_fa2_C::varlen_fwd",
"duration_sum_us": 176996.086,
"duration_us": {
"n": 3456,
"p50": 33.433,
"p95": 117.412,
"mean": 51.214145254629635,
"min": 18.375,
"max": 4124.091
}
},
{
"category": "cpu_op",
"name": "vllm::unified_kv_cache_update",
"duration_sum_us": 165200.251,
"duration_us": {
"n": 3456,
"p50": 37.711,
"p95": 92.185,
"mean": 47.80099855324074,
"min": 23.675,
"max": 11344.783
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 139704.374,
"duration_us": {
"n": 9765,
"p50": 10.56,
"p95": 32.743,
"mean": 14.306643522785459,
"min": 0.601,
"max": 1552.228
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 137013.92,
"duration_us": {
"n": 7968,
"p50": 13.636,
"p95": 38.293,
"mean": 17.195522088353414,
"min": 0.13,
"max": 1562.447
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 130793.102,
"duration_us": {
"n": 7488,
"p50": 13.396,
"p95": 38.112,
"mean": 17.46702751068376,
"min": 8.045,
"max": 1561.555
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 99321.441,
"duration_us": {
"n": 3,
"p50": 34808.033,
"p95": 36062.434,
"mean": 33107.147000000004,
"min": 28450.974,
"max": 36062.434
}
},
{
"category": "user_annotation",
"name": "execute_context_1(18)_generation_0(0)",
"duration_sum_us": 93584.686,
"duration_us": {
"n": 3,
"p50": 34106.638,
"p95": 34504.985,
"mean": 31194.895333333334,
"min": 24973.063,
"max": 34504.985
}
},
{
"category": "cpu_op",
"name": "_C::fused_add_rms_norm",
"duration_sum_us": 81865.368,
"duration_us": {
"n": 6912,
"p50": 9.629,
"p95": 25.438,
"mean": 11.843947916666666,
"min": 6.372,
"max": 477.797
}
},
{
"category": "cpu_op",
"name": "_C_cache_ops::reshape_and_cache_flash",
"duration_sum_us": 62589.834,
"duration_us": {
"n": 3456,
"p50": 12.394,
"p95": 33.264,
"mean": 18.110484375000002,
"min": 8.225,
"max": 11211.29
}
},
{
"category": "cpu_op",
"name": "aten::mul",
"duration_sum_us": 50819.894,
"duration_us": {
"n": 3456,
"p50": 12.023,
"p95": 25.659,
"mean": 14.704830439814815,
"min": 7.404,
"max": 3471.447
}
},
{
"category": "cpu_op",
"name": "_C::gelu_and_mul",
"duration_sum_us": 49811.896,
"duration_us": {
"n": 3456,
"p50": 9.177,
"p95": 22.893,
"mean": 14.413164351851853,
"min": 5.811,
"max": 12140.8
}
},
{
"category": "cpu_op",
"name": "_C::rotary_embedding",
"duration_sum_us": 48813.654,
"duration_us": {
"n": 3456,
"p50": 10.85,
"p95": 28.835,
"mean": 14.124321180555556,
"min": 7.083,
"max": 3405.417
}
},
{
"category": "cpu_op",
"name": "aten::sigmoid",
"duration_sum_us": 42525.64,
"duration_us": {
"n": 3456,
"p50": 10.74,
"p95": 21.13,
"mean": 12.304872685185185,
"min": 6.523,
"max": 572.547
}
},
{
"category": "kernel",
"name": "void flash::flash_fwd_splitkv_kernel<Flash_fwd_kernel_traits<256, 64, 64, 4, false, false, cutlass::bfloat16_t, Flash_kernel_traits<256, 64, 64, 4, cutlass::bfloat16_t> >, true, false, false, false, true, false, false, false>(flash::Flash_fwd_params)",
"duration_sum_us": 41230.898,
"duration_us": {
"n": 2619,
"p50": 10.593,
"p95": 18.144,
"mean": 15.742992745322642,
"min": 7.648,
"max": 5208.423
}
},
{
"category": "overhead",
"name": "Buffer Flush",
"duration_sum_us": 41112.648,
"duration_us": {
"n": 6,
"p50": 518.317,
"p95": 17002.037,
"mean": 6852.108,
"min": 215.84,
"max": 17002.037
}
},
{
"category": "cpu_op",
"name": "aten::t",
"duration_sum_us": 39718.074,
"duration_us": {
"n": 17376,
"p50": 1.664,
"p95": 4.389,
"mean": 2.285800759668508,
"min": 0.892,
"max": 188.398
}
},
{
"category": "cpu_op",
"name": "aten::split",
"duration_sum_us": 37431.756,
"duration_us": {
"n": 6912,
"p50": 3.787,
"p95": 7.925,
"mean": 5.415473958333333,
"min": 2.384,
"max": 356.111
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 36820.371,
"duration_us": {
"n": 2181,
"p50": 13.896,
"p95": 38.904,
"mean": 16.882334250343877,
"min": 3.847,
"max": 339.53
}
},
{
"category": "cpu_op",
"name": "aten::empty",
"duration_sum_us": 34376.117,
"duration_us": {
"n": 17472,
"p50": 1.653,
"p95": 3.397,
"mean": 1.9674975389194138,
"min": 0.971,
"max": 304.248
}
},
{
"category": "cpu_op",
"name": "aten::empty_strided",
"duration_sum_us": 34265.595,
"duration_us": {
"n": 11877,
"p50": 2.315,
"p95": 6.051,
"mean": 2.8850378883556456,
"min": 1.262,
"max": 256.897
}
},
{
"category": "cpu_op",
"name": "aten::transpose",
"duration_sum_us": 33385.55,
"duration_us": {
"n": 26799,
"p50": 0.882,
"p95": 2.555,
"mean": 1.2457759617896191,
"min": 0.431,
"max": 173.841
}
},
{
"category": "kernel",
"name": "std::enable_if<(((8)>(0)))&&vllm::_typeConvert<c10::BFloat16>::exists, void>::type vllm::fused_add_rms_norm_kernel<c10::BFloat16, 8, true>(c10::BFloat16*, long, c10::BFloat16*, c10::BFloat16 const*, float, int, int, long)",
"duration_sum_us": 29832.223,
"duration_us": {
"n": 6912,
"p50": 3.297,
"p95": 5.824,
"mean": 4.316004484953704,
"min": 1.92,
"max": 2003.892
}
},
{
"category": "cpu_op",
"name": "aten::slice",
"duration_sum_us": 29713.287,
"duration_us": {
"n": 26709,
"p50": 0.682,
"p95": 1.933,
"mean": 1.1124821970122432,
"min": 0.41,
"max": 340.541
}
},
{
"category": "cpu_op",
"name": "aten::new_empty",
"duration_sum_us": 27681.109,
"duration_us": {
"n": 10368,
"p50": 2.024,
"p95": 4.418,
"mean": 2.669860050154321,
"min": 1.333,
"max": 449.483
}
},
{
"category": "cpu_op",
"name": "aten::narrow",
"duration_sum_us": 24699.753,
"duration_us": {
"n": 13824,
"p50": 1.123,
"p95": 3.376,
"mean": 1.7867298177083333,
"min": 0.761,
"max": 343.267
}
},
{
"category": "cpu_op",
"name": "aten::as_strided",
"duration_sum_us": 24658.427,
"duration_us": {
"n": 78282,
"p50": 0.23,
"p95": 0.761,
"mean": 0.3149948519455303,
"min": 0.05,
"max": 152.029
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_128x2_tn_align8>(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_128x2_tn_align8::Params)",
"duration_sum_us": 19339.032,
"duration_us": {
"n": 108,
"p50": 170.053,
"p95": 200.805,
"mean": 179.0651111111111,
"min": 159.909,
"max": 534.606
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_tensorop_bf16_s16816gemm_relu_bf16_128x64_64x3_tn_align8>(cutlass_80_tensorop_bf16_s16816gemm_relu_bf16_128x64_64x3_tn_align8::Params)",
"duration_sum_us": 18074.894,
"duration_us": {
"n": 216,
"p50": 82.849,
"p95": 95.619,
"mean": 83.68006481481481,
"min": 34.081,
"max": 1841.807
}
},
{
"category": "cpu_op",
"name": "aten::empty_like",
"duration_sum_us": 17296.837,
"duration_us": {
"n": 4293,
"p50": 3.106,
"p95": 6.733,
"mean": 4.029079198695551,
"min": 2.023,
"max": 277.306
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-trace\\traces\\20260922-185848_e2e-stage0\\trace_rank0.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 122,
"duration_sum_us": 12304399.495117188,
"busy_union_us": 12304399.495117188
},
"cpu_op": {
"events": 59554,
"duration_sum_us": 8906437.828125,
"busy_union_us": 3168870.0078125
},
"overhead": {
"events": 36,
"duration_sum_us": 19140.3017578125,
"busy_union_us": 19140.3017578125
},
"cuda_runtime": {
"events": 4781,
"duration_sum_us": 5376328.5263671875,
"busy_union_us": 2793137.51953125
},
"kernel": {
"events": 54380,
"duration_sum_us": 2817254.326171875,
"busy_union_us": 2817175.0947265625
},
"gpu_memcpy": {
"events": 1077,
"duration_sum_us": 3044.5390625,
"busy_union_us": 3044.5390625
},
"cuda_driver": {
"events": 289,
"duration_sum_us": 2079.8017578125,
"busy_union_us": 2079.8017578125
},
"gpu_user_annotation": {
"events": 30,
"duration_sum_us": 2949605.2578125,
"busy_union_us": 2949605.2578125
},
"Trace": {
"events": 1,
"duration_sum_us": 13212500.400390625,
"busy_union_us": 13212500.400390625
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 13212500.4,
"duration_us": {
"n": 1,
"p50": 13212500.4,
"p95": 13212500.4,
"mean": 13212500.4,
"min": 13212500.4,
"max": 13212500.4
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 11710790.5,
"duration_us": {
"n": 116,
"p50": 89821.8,
"p95": 167032.0,
"mean": 100955.09051724138,
"min": 13606.3,
"max": 328952.7
}
},
{
"category": "cuda_runtime",
"name": "cudaEventSynchronize",
"duration_sum_us": 4598600.0,
"duration_us": {
"n": 115,
"p50": 22.8,
"p95": 115605.2,
"mean": 39987.82608695652,
"min": 1.7,
"max": 160646.4
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 2907728.214,
"duration_us": {
"n": 29,
"p50": 91709.378,
"p95": 153430.373,
"mean": 100266.49013793105,
"min": 46463.684,
"max": 167890.296
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 2845682.4,
"duration_us": {
"n": 4853,
"p50": 13.4,
"p95": 90.3,
"mean": 586.3759324129404,
"min": 0.3,
"max": 124829.2
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 2788886.0,
"duration_us": {
"n": 2631,
"p50": 2.3,
"p95": 168.1,
"mean": 1060.0098821740783,
"min": 0.0,
"max": 124850.2
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 2784892.7,
"duration_us": {
"n": 1146,
"p50": 29.4,
"p95": 21892.3,
"mean": 2430.098342059337,
"min": 2.5,
"max": 124847.5
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I67cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_64x2_tn_align8EEvNT_6ParamsE",
"duration_sum_us": 1325890.581,
"duration_us": {
"n": 6507,
"p50": 139.491,
"p95": 324.263,
"mean": 203.7637284462886,
"min": 89.474,
"max": 5205.722
}
},
{
"category": "cuda_runtime",
"name": "cudaStreamSynchronize",
"duration_sum_us": 658950.7,
"duration_us": {
"n": 37,
"p50": 21179.0,
"p95": 29413.0,
"mean": 17809.47837837838,
"min": 2.5,
"max": 29668.2
}
},
{
"category": "kernel",
"name": "_ZN8internal5gemvx6kernelIii13__nv_bfloat16S2_S2_fLb0ELb1ELb1ELb0ELi6ELb0E18cublasGemvParamsExIi30cublasGemvTensorStridedBatchedIKS2_ES6_S4_IS2_EfEEENSt9enable_ifIXntT5_EvE4typeET11_",
"duration_sum_us": 624599.958,
"duration_us": {
"n": 3593,
"p50": 146.467,
"p95": 270.343,
"mean": 173.83800667965488,
"min": 89.89,
"max": 3258.444
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_0(0)",
"duration_sum_us": 517878.30000000005,
"duration_us": {
"n": 3,
"p50": 182859.0,
"p95": 191915.6,
"mean": 172626.1,
"min": 143103.7,
"max": 191915.6
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I68cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_128x2_tn_align8EEvNT_6ParamsE",
"duration_sum_us": 470950.459,
"duration_us": {
"n": 2170,
"p50": 181.924,
"p95": 271.238,
"mean": 217.02786129032256,
"min": 175.364,
"max": 6000.556
}
},
{
"category": "cuda_runtime",
"name": "cudaGraphLaunch",
"duration_sum_us": 79170.5,
"duration_us": {
"n": 56,
"p50": 1036.3,
"p95": 2863.7,
"mean": 1413.7589285714287,
"min": 462.3,
"max": 3009.3
}
},
{
"category": "user_annotation",
"name": "execute_context_1(19)_generation_0(0)",
"duration_sum_us": 75730.70000000001,
"duration_us": {
"n": 3,
"p50": 21779.7,
"p95": 34530.3,
"mean": 25243.56666666667,
"min": 19420.7,
"max": 34530.3
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I42cutlass_80_simt_sgemm_128x32_8x5_nn_align1EEvNT_6ParamsE",
"duration_sum_us": 56782.787,
"duration_us": {
"n": 2142,
"p50": 15.393,
"p95": 72.609,
"mean": 26.509237628384685,
"min": 8.801,
"max": 1177.5
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(19)_generation_0(0)",
"duration_sum_us": 41877.045,
"duration_us": {
"n": 1,
"p50": 41877.045,
"p95": 41877.045,
"mean": 41877.045,
"min": 41877.045,
"max": 41877.045
}
},
{
"category": "cpu_op",
"name": "aten::slice",
"duration_sum_us": 34583.7,
"duration_us": {
"n": 9352,
"p50": 2.1,
"p95": 7.7,
"mean": 3.6980004277159964,
"min": 0.7,
"max": 228.8
}
},
{
"category": "kernel",
"name": "_Z20magma_sgemmEx_kernelIfffLb0ELb0ELi6ELi3ELi5ELi3ELi3EEviii6TensoriS0_iS0_iS0_iiiPKT_S3_S1_S1_i18cublasLtEpilogue_tiPKvx",
"duration_sum_us": 31418.819,
"duration_us": {
"n": 2141,
"p50": 9.408,
"p95": 36.833,
"mean": 14.674833722559551,
"min": 5.089,
"max": 702.64
}
},
{
"category": "cpu_op",
"name": "aten::fill_",
"duration_sum_us": 29675.3,
"duration_us": {
"n": 1553,
"p50": 9.2,
"p95": 63.5,
"mean": 19.108370895041855,
"min": 0.7,
"max": 2150.2
}
},
{
"category": "cpu_op",
"name": "aten::sort",
"duration_sum_us": 29632.4,
"duration_us": {
"n": 238,
"p50": 111.8,
"p95": 221.6,
"mean": 124.50588235294119,
"min": 73.5,
"max": 448.6
}
},
{
"category": "kernel",
"name": "_ZN8cublasLt19splitKreduce_kernelILi32ELi16Ei13__nv_bfloat16S1_fS1_Lb0ES1_S1_S1_Lb1ELb0ELb0ELb0EEEvNS_18cublasSplitKParamsIT4_EEPKT2_PKT8_PT7_PT3_PKS3_SG_PKT9_S7_PSH_PvxPS3_PiSM_SM_SG_SG_SG_SG_SG_",
"duration_sum_us": 25247.384000000002,
"duration_us": {
"n": 4338,
"p50": 4.576,
"p95": 12.257,
"mean": 5.82005163669894,
"min": 1.888,
"max": 64.098
}
},
{
"category": "kernel",
"name": "_ZN5flash24flash_fwd_splitkv_kernelI23Flash_fwd_kernel_traitsILi128ELi64ELi128ELi4ELb0ELb0EN7cutlass10bfloat16_tE19Flash_kernel_traitsILi128ELi64ELi128ELi4ES3_EELb0ELb0ELb0ELb0ELb1ELb0ELb1ELb0EEEvNS_16Flash_fwd_paramsE",
"duration_sum_us": 24629.815,
"duration_us": {
"n": 784,
"p50": 20.513,
"p95": 84.13,
"mean": 31.415580357142854,
"min": 14.273,
"max": 87.138
}
},
{
"category": "cpu_op",
"name": "aten::cat",
"duration_sum_us": 23662.7,
"duration_us": {
"n": 604,
"p50": 24.8,
"p95": 117.1,
"mean": 39.17665562913908,
"min": 9.6,
"max": 329.4
}
},
{
"category": "cpu_op",
"name": "## Call CompiledFxGraph None ##",
"duration_sum_us": 22029.0,
"duration_us": {
"n": 87,
"p50": 222.4,
"p95": 403.9,
"mean": 253.20689655172413,
"min": 155.5,
"max": 617.1
}
},
{
"category": "kernel",
"name": "triton_per_fused__to_copy__unsafe_view_add_cat_clone_index_mean_mul_neg_pow_rsqrt_slice_split_transpose_unsqueeze_view_1",
"duration_sum_us": 20781.403,
"duration_us": {
"n": 2142,
"p50": 8.896,
"p95": 16.224,
"mean": 9.701868814192343,
"min": 4.832,
"max": 19.745
}
},
{
"category": "kernel",
"name": "triton_red_fused__to_copy__unsafe_view_add_cat_clone_expand_index_mean_mul_neg_pow_rsqrt_slice_split_transpose_unsqueeze_view_2",
"duration_sum_us": 20543.918999999998,
"duration_us": {
"n": 2142,
"p50": 7.136,
"p95": 18.049,
"mean": 9.590998599439775,
"min": 2.816,
"max": 2281.749
}
},
{
"category": "cuda_runtime",
"name": "cudaLaunchKernel",
"duration_sum_us": 20360.6,
"duration_us": {
"n": 1535,
"p50": 7.2,
"p95": 43.8,
"mean": 13.264234527687295,
"min": 3.3,
"max": 2134.1
}
},
{
"category": "cpu_op",
"name": "aten::index",
"duration_sum_us": 19423.0,
"duration_us": {
"n": 241,
"p50": 66.2,
"p95": 146.3,
"mean": 80.59336099585062,
"min": 18.8,
"max": 441.4
}
},
{
"category": "kernel",
"name": "_ZN2at6native6sbtopk10gatherTopKIN3c108BFloat16EjLi1ELb0EEEvNS_4cuda6detail10TensorInfoIKT_T0_EESA_SA_bSA_SA_NS7_IS8_SA_EESA_NS7_IxSA_EESA_PS8_",
"duration_sum_us": 18658.516,
"duration_us": {
"n": 428,
"p50": 23.489,
"p95": 122.531,
"mean": 43.594663551401865,
"min": 15.104,
"max": 653.168
}
},
{
"category": "cpu_op",
"name": "aten::clone",
"duration_sum_us": 18299.8,
"duration_us": {
"n": 723,
"p50": 20.0,
"p95": 60.2,
"mean": 25.310926694329183,
"min": 5.7,
"max": 338.0
}
},
{
"category": "cpu_op",
"name": "aten::mm",
"duration_sum_us": 17397.3,
"duration_us": {
"n": 455,
"p50": 19.8,
"p95": 108.8,
"mean": 38.23582417582418,
"min": 8.6,
"max": 308.9
}
},
{
"category": "kernel",
"name": "_ZN5flash32flash_fwd_splitkv_combine_kernelI23Flash_fwd_kernel_traitsILi128ELi64ELi128ELi4ELb0ELb0EN7cutlass10bfloat16_tE19Flash_kernel_traitsILi128ELi64ELi128ELi4ES3_EELi4ELi4ELb1EEEvNS_16Flash_fwd_paramsE",
"duration_sum_us": 16541.874,
"duration_us": {
"n": 784,
"p50": 13.12,
"p95": 52.641,
"mean": 21.099329081632654,
"min": 7.744,
"max": 1228.093
}
},
{
"category": "overhead",
"name": "Activity Buffer Request",
"duration_sum_us": 16391.3,
"duration_us": {
"n": 34,
"p50": 441.0,
"p95": 751.0,
"mean": 482.0970588235294,
"min": 399.9,
"max": 754.2
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 15650.1,
"duration_us": {
"n": 1077,
"p50": 9.0,
"p95": 46.3,
"mean": 14.531197771587744,
"min": 4.0,
"max": 233.1
}
},
{
"category": "cpu_op",
"name": "aten::index_copy_",
"duration_sum_us": 15022.2,
"duration_us": {
"n": 232,
"p50": 62.8,
"p95": 157.7,
"mean": 64.75086206896552,
"min": 17.7,
"max": 330.0
}
},
{
"category": "cpu_op",
"name": "aten::linear",
"duration_sum_us": 14007.2,
"duration_us": {
"n": 137,
"p50": 88.5,
"p95": 170.0,
"mean": 102.24233576642337,
"min": 18.2,
"max": 377.7
}
},
{
"category": "cpu_op",
"name": "aten::empty_strided",
"duration_sum_us": 12025.9,
"duration_us": {
"n": 2310,
"p50": 3.5,
"p95": 13.7,
"mean": 5.2060173160173155,
"min": 0.3,
"max": 52.3
}
},
{
"category": "kernel",
"name": "triton_per_fused__safe_softmax_add_ones_prepare_softmax_online_scalar_tensor_tril_view_where_3",
"duration_sum_us": 11643.498,
"duration_us": {
"n": 2142,
"p50": 4.032,
"p95": 9.472,
"mean": 5.435806722689075,
"min": 1.568,
"max": 1217.021
}
},
{
"category": "cpu_op",
"name": "aten::matmul",
"duration_sum_us": 11110.6,
"duration_us": {
"n": 119,
"p50": 78.1,
"p95": 143.7,
"mean": 93.36638655462185,
"min": 28.5,
"max": 321.1
}
},
{
"category": "kernel",
"name": "_ZN2at6native22warpMergeSortKVInPlaceILin2ELin1ELi128ELi16EN3c108BFloat16ExNS0_4GTOpIS3_Lb1EEEjLi32EEEvNS_4cuda6detail10TensorInfoIT3_T6_EESA_SA_SA_NS8_IT4_SA_EESA_T5_S9_",
"duration_sum_us": 10662.185,
"duration_us": {
"n": 428,
"p50": 14.304,
"p95": 52.929,
"mean": 24.911647196261683,
"min": 8.224,
"max": 1339.647
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-windows-trace\\traces\\20260922-185848_e2e-stage1\\trace_rank0.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 6380,
"duration_sum_us": 920313.8681640625,
"busy_union_us": 920313.8681640625
},
"cpu_op": {
"events": 4851,
"duration_sum_us": 7392848.85546875,
"busy_union_us": 2479212.1025390625
},
"overhead": {
"events": 1,
"duration_sum_us": 727.099609375,
"busy_union_us": 727.099609375
},
"cuda_runtime": {
"events": 19888,
"duration_sum_us": 2563179.6865234375,
"busy_union_us": 2563179.6865234375
},
"gpu_memcpy": {
"events": 243,
"duration_sum_us": 1211.548828125,
"busy_union_us": 1211.548828125
},
"kernel": {
"events": 10977,
"duration_sum_us": 2329885.2548828125,
"busy_union_us": 2329828.775390625
},
"gpu_memset": {
"events": 48,
"duration_sum_us": 76.5126953125,
"busy_union_us": 76.5126953125
},
"gpu_user_annotation": {
"events": 15,
"duration_sum_us": 2505547.587890625,
"busy_union_us": 2505547.587890625
},
"Trace": {
"events": 1,
"duration_sum_us": 27011304.0,
"busy_union_us": 27011304.0
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 27011304.0,
"duration_us": {
"n": 1,
"p50": 27011304.0,
"p95": 27011304.0,
"mean": 27011304.0,
"min": 27011304.0,
"max": 27011304.0
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 2459518.7,
"duration_us": {
"n": 666,
"p50": 1.2,
"p95": 74.1,
"mean": 3692.9710210210214,
"min": 0.5,
"max": 317404.2
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 2455931.8,
"duration_us": {
"n": 243,
"p50": 10.9,
"p95": 70200.7,
"mean": 10106.715226337448,
"min": 5.0,
"max": 317371.2
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 2455510.5,
"duration_us": {
"n": 165,
"p50": 6.9,
"p95": 109711.7,
"mean": 14881.881818181819,
"min": 0.1,
"max": 317434.7
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 2454885.7,
"duration_us": {
"n": 75,
"p50": 30.1,
"p95": 263335.1,
"mean": 32731.809333333335,
"min": 6.9,
"max": 317430.9
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_0(0)",
"duration_sum_us": 854940.8,
"duration_us": {
"n": 6365,
"p50": 102.0,
"p95": 293.5,
"mean": 134.31905734485468,
"min": 30.7,
"max": 1182.4
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(256)_generation_0(0)",
"duration_sum_us": 780462.085,
"duration_us": {
"n": 3,
"p50": 240451.01,
"p95": 320077.399,
"mean": 260154.02833333332,
"min": 219933.676,
"max": 320077.399
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(64)_generation_0(0)",
"duration_sum_us": 563678.599,
"duration_us": {
"n": 4,
"p50": 91434.88,
"p95": 276983.483,
"mean": 140919.64975,
"min": 72505.351,
"max": 276983.483
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(128)_generation_0(0)",
"duration_sum_us": 373658.788,
"duration_us": {
"n": 3,
"p50": 112414.795,
"p95": 152116.407,
"mean": 124552.92933333333,
"min": 109127.586,
"max": 152116.407
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(160)_generation_0(0)",
"duration_sum_us": 291740.902,
"duration_us": {
"n": 1,
"p50": 291740.902,
"p95": 291740.902,
"mean": 291740.902,
"min": 291740.902,
"max": 291740.902
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(144)_generation_0(0)",
"duration_sum_us": 265968.47,
"duration_us": {
"n": 1,
"p50": 265968.47,
"p95": 265968.47,
"mean": 265968.47,
"min": 265968.47,
"max": 265968.47
}
},
{
"category": "kernel",
"name": "_ZN17cutlass__5x_cudnn6KernelI65cutlass_tensorop_s1688fprop_optimized_tf32_64x64_32x5_nhwc_align4EEvNT_6ParamsE",
"duration_sum_us": 233077.019,
"duration_us": {
"n": 120,
"p50": 577.582,
"p95": 5954.284,
"mean": 1942.3084916666667,
"min": 34.337,
"max": 11253.544
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(32)_generation_0(0)",
"duration_sum_us": 230038.744,
"duration_us": {
"n": 3,
"p50": 74022.73,
"p95": 91382.462,
"mean": 76679.58133333334,
"min": 64633.552,
"max": 91382.462
}
},
{
"category": "kernel",
"name": "_ZN5cudnn19engines_precompiled16nchwToNhwcKernelIfffLb0ELb1EL21cudnnKernelDataType_t2EEEvNS0_18nchw2nhwc_params_tIT1_EEPKT_PT0_",
"duration_sum_us": 204347.104,
"duration_us": {
"n": 336,
"p50": 63.394,
"p95": 4226.819,
"mean": 608.1759047619047,
"min": 4.096,
"max": 9423.517
}
},
{
"category": "kernel",
"name": "_Z26precomputed_convolve_sgemmIfLi1024ELi5ELi5ELi4ELi3ELi3ELi1ELb0EEviiiPKT_iPS0_S2_18kernel_conv_paramsyiffibbS2_S2_Pi",
"duration_sum_us": 191147.012,
"duration_us": {
"n": 33,
"p50": 4581.227,
"p95": 14234.094,
"mean": 5792.333696969697,
"min": 497.612,
"max": 15142.692
}
},
{
"category": "kernel",
"name": "_ZN5cudnn6detail12dgrad_engineIfLi512ELi6ELi5ELi3ELi3ELi3ELb0EEEviiiPKT_iS4_iPS2_18kernel_grad_paramsyiyifiii",
"duration_sum_us": 154151.615,
"duration_us": {
"n": 45,
"p50": 2641.887,
"p95": 9885.224,
"mean": 3425.5914444444443,
"min": 85.218,
"max": 18971.612
}
},
{
"category": "kernel",
"name": "_ZN2at6native29vectorized_elementwise_kernelILi4ENS0_15CUDAFunctor_addIfEESt5arrayIPcLy3EEEEviT0_T1_",
"duration_sum_us": 143312.61000000002,
"duration_us": {
"n": 645,
"p50": 5.568,
"p95": 1186.908,
"mean": 222.19009302325583,
"min": 1.568,
"max": 9725.315
}
},
{
"category": "kernel",
"name": "_kernel",
"duration_sum_us": 142784.588,
"duration_us": {
"n": 435,
"p50": 19.489,
"p95": 2811.489,
"mean": 328.240432183908,
"min": 4.16,
"max": 7893.337
}
},
{
"category": "kernel",
"name": "_Z23implicit_convolve_sgemmIffLi128ELi5ELi5ELi3ELi3ELi3ELi1ELb0ELb0ELb1EEviiiPKT_iPT0_S2_18kernel_conv_paramsyiffiPKS3_S7_bbii",
"duration_sum_us": 136572.834,
"duration_us": {
"n": 30,
"p50": 3127.594,
"p95": 15143.652,
"mean": 4552.4278,
"min": 189.22,
"max": 25229.007
}
},
{
"category": "kernel",
"name": "_Z23implicit_convolve_sgemmIffLi128ELi5ELi5ELi3ELi3ELi3ELi1ELb0ELb0ELb0EEviiiPKT_iPT0_S2_18kernel_conv_paramsyiffiPKS3_S7_bbii",
"duration_sum_us": 133259.189,
"duration_us": {
"n": 30,
"p50": 3179.626,
"p95": 11115.973,
"mean": 4441.972966666667,
"min": 398.666,
"max": 11796.629
}
},
{
"category": "kernel",
"name": "_ZN2at6native18elementwise_kernelILi128ELi2EZNS0_22gpu_kernel_impl_nocastINS0_15CUDAFunctor_addIfEEEEvRNS_18TensorIteratorBaseERKT_EUliE_EEviT1_",
"duration_sum_us": 131171.26,
"duration_us": {
"n": 780,
"p50": 10.208,
"p95": 278.022,
"mean": 168.16828205128206,
"min": 1.888,
"max": 6502.232
}
},
{
"category": "kernel",
"name": "sm80_xmma_dgrad_implicit_gemm_indexed_tf32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x16_stage4_warpsize2x2x1_g1_tensor16x8x8_strided_execute_kernel__5x_cudnn",
"duration_sum_us": 75365.961,
"duration_us": {
"n": 9,
"p50": 7516.368,
"p95": 17592.573,
"mean": 8373.995666666666,
"min": 507.18,
"max": 17592.573
}
},
{
"category": "kernel",
"name": "_ZN2at6native18elementwise_kernelILi128ELi2EZNS0_22gpu_kernel_impl_nocastIZZZNS0_23direct_copy_kernel_cudaERNS_18TensorIteratorBaseEENKUlvE1_clEvENKUlvE5_clEvEUlfE_EEvS4_RKT_EUliE_EEviT1_",
"duration_sum_us": 73221.556,
"duration_us": {
"n": 549,
"p50": 7.52,
"p95": 333.576,
"mean": 133.3725974499089,
"min": 1.985,
"max": 6940.419
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I52cutlass_80_tensorop_s1688gemm_256x128_16x3_nn_align4EEvNT_6ParamsE",
"duration_sum_us": 69999.686,
"duration_us": {
"n": 54,
"p50": 228.741,
"p95": 6225.714,
"mean": 1296.2904814814815,
"min": 33.536,
"max": 7978.043
}
},
{
"category": "kernel",
"name": "memcpy128",
"duration_sum_us": 65516.163,
"duration_us": {
"n": 144,
"p50": 15.36,
"p95": 3483.89,
"mean": 454.9733541666667,
"min": 3.904,
"max": 6399.062
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I42cutlass_80_simt_sgemm_128x64_8x5_tn_align1EEvNT_6ParamsE",
"duration_sum_us": 56742.585,
"duration_us": {
"n": 219,
"p50": 66.434,
"p95": 2632.062,
"mean": 259.09856164383564,
"min": 52.865,
"max": 6294.132
}
},
{
"category": "kernel",
"name": "_ZN2at6native29vectorized_elementwise_kernelILi4ENS0_11FillFunctorIfEESt5arrayIPcLy1EEEEviT0_T1_",
"duration_sum_us": 54289.676,
"duration_us": {
"n": 627,
"p50": 3.968,
"p95": 236.934,
"mean": 86.58640510366826,
"min": 1.184,
"max": 9155.446
}
},
{
"category": "kernel",
"name": "_ZN17cutlass__5x_cudnn6KernelI66cutlass_tensorop_s1688dgrad_optimized_tf32_128x64_32x3_nhwc_align4EEvNT_6ParamsE",
"duration_sum_us": 41469.901,
"duration_us": {
"n": 15,
"p50": 746.578,
"p95": 10423.444,
"mean": 2764.6600666666664,
"min": 139.363,
"max": 10423.444
}
},
{
"category": "cuda_runtime",
"name": "cudaEventSynchronize",
"duration_sum_us": 40705.9,
"duration_us": {
"n": 6395,
"p50": 4.0,
"p95": 17.5,
"mean": 6.3652697419859265,
"min": 0.7,
"max": 93.1
}
},
{
"category": "kernel",
"name": "_Z16gemmSN_TN_kernelIfLi128ELi16ELi2ELi4ELi6ELi7ELb0E30cublasGemvTensorStridedBatchedIKfES2_S0_IfEEv22cublasGemmSmallNParamsIT7_T8_T9_T_E",
"duration_sum_us": 36720.037,
"duration_us": {
"n": 177,
"p50": 59.617,
"p95": 1055.321,
"mean": 207.45783615819207,
"min": 47.521,
"max": 5828.553
}
},
{
"category": "kernel",
"name": "_ZN2at6native18elementwise_kernelILi128ELi2EZNS0_22gpu_kernel_impl_nocastINS0_13BinaryFunctorIfffNS0_15binary_internal10MulFunctorIfEEEEEEvRNS_18TensorIteratorBaseERKT_EUliE_EEviT1_",
"duration_sum_us": 30402.853,
"duration_us": {
"n": 1260,
"p50": 7.68,
"p95": 11.649,
"mean": 24.129248412698413,
"min": 1.76,
"max": 5297.181
}
},
{
"category": "kernel",
"name": "_Z18sgemm_largek_lds64ILb1ELb0ELi5ELi5ELi4ELi4ELi4ELi34EEvPfPKfS2_iiiiiiS2_S2_ffiiPiS3_",
"duration_sum_us": 30029.248,
"duration_us": {
"n": 24,
"p50": 554.061,
"p95": 4735.055,
"mean": 1251.2186666666666,
"min": 71.33,
"max": 4873.746
}
},
{
"category": "kernel",
"name": "_ZN2at6native18elementwise_kernelILi128ELi2EZNS0_22gpu_kernel_impl_nocastINS0_13BinaryFunctorIfffNS0_15binary_internal10DivFunctorIfEEEEEEvRNS_18TensorIteratorBaseERKT_EUliE_EEviT1_",
"duration_sum_us": 29066.1,
"duration_us": {
"n": 240,
"p50": 53.762,
"p95": 73.346,
"mean": 121.10875,
"min": 49.122,
"max": 5068.119
}
},
{
"category": "cuda_runtime",
"name": "cudaEventRecordWithFlags",
"duration_sum_us": 24843.4,
"duration_us": {
"n": 6410,
"p50": 2.7,
"p95": 8.9,
"mean": 3.875725429017161,
"min": 0.6,
"max": 60.9
}
},
{
"category": "kernel",
"name": "_Z16gemmSN_TN_kernelIfLi128ELi16ELi2ELi4ELi14ELi15ELb0E30cublasGemvTensorStridedBatchedIKfES2_S0_IfEEv22cublasGemmSmallNParamsIT7_T8_T9_T_E",
"duration_sum_us": 23089.019,
"duration_us": {
"n": 171,
"p50": 56.993,
"p95": 82.882,
"mean": 135.02350292397662,
"min": 48.993,
"max": 2680.191
}
},
{
"category": "user_annotation",
"name": "execute_context_1(32)_generation_0(0)",
"duration_sum_us": 22742.6,
"duration_us": {
"n": 3,
"p50": 4521.4,
"p95": 13719.3,
"mean": 7580.866666666666,
"min": 4501.9,
"max": 13719.3
}
},
{
"category": "kernel",
"name": "_ZN5cudnn19engines_precompiled16nhwcToNchwKernelIfffLb1ELb0EL21cudnnKernelDataType_t0EEEvNS0_18nhwc2nchw_params_tIT1_EEPKT_PT0_",
"duration_sum_us": 21749.371,
"duration_us": {
"n": 168,
"p50": 14.177,
"p95": 394.057,
"mean": 129.46054166666667,
"min": 2.624,
"max": 3750.168
}
},
{
"category": "kernel",
"name": "_ZN7cutlass7Kernel2I42cutlass_80_simt_sgemm_128x32_8x5_tn_align1EEvNT_6ParamsE",
"duration_sum_us": 21679.04,
"duration_us": {
"n": 120,
"p50": 74.529,
"p95": 851.028,
"mean": 180.65866666666668,
"min": 51.969,
"max": 2927.812
}
},
{
"category": "kernel",
"name": "_Z16gemmSN_TN_kernelIfLi128ELi16ELi2ELi4ELi2ELi2ELb1E30cublasGemvTensorStridedBatchedIKfES2_S0_IfEEv22cublasGemmSmallNParamsIT7_T8_T9_T_E",
"duration_sum_us": 20449.285,
"duration_us": {
"n": 171,
"p50": 55.393,
"p95": 82.85,
"mean": 119.5864619883041,
"min": 48.034,
"max": 4486.633
}
},
{
"category": "cuda_runtime",
"name": "cudaGraphLaunch",
"duration_sum_us": 18002.2,
"duration_us": {
"n": 15,
"p50": 1229.0,
"p95": 1340.4,
"mean": 1200.1466666666668,
"min": 1068.4,
"max": 1340.4
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-trace\\traces\\20260922-185614_e2e-stage0\\trace_rank0.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 123,
"duration_sum_us": 13282071.109375,
"busy_union_us": 13282071.109375
},
"cpu_op": {
"events": 60009,
"duration_sum_us": 9767336.748046875,
"busy_union_us": 3478677.3994140625
},
"overhead": {
"events": 332,
"duration_sum_us": 593661.8564453125,
"busy_union_us": 593661.8564453125
},
"cuda_runtime": {
"events": 17725,
"duration_sum_us": 25891282.77734375,
"busy_union_us": 13563986.081054688
},
"gpu_memcpy": {
"events": 4356,
"duration_sum_us": 14788.46875,
"busy_union_us": 14788.46875
},
"kernel": {
"events": 196938,
"duration_sum_us": 13561556.567382812,
"busy_union_us": 13561423.46484375
},
"cuda_driver": {
"events": 897,
"duration_sum_us": 29843.9072265625,
"busy_union_us": 29843.9072265625
},
"gpu_user_annotation": {
"events": 120,
"duration_sum_us": 14152552.818359375,
"busy_union_us": 14152552.818359375
},
"Trace": {
"events": 1,
"duration_sum_us": 14340331.1171875,
"busy_union_us": 14340331.1171875
}
},
"largest_ops_by_duration_sum": [
{
"category": "cuda_runtime",
"name": "cudaEventSynchronize",
"duration_sum_us": 22308482.111,
"duration_us": {
"n": 480,
"p50": 9.478,
"p95": 129331.156,
"mean": 46476.00439791667,
"min": 2.535,
"max": 417383.189
}
},
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 14340331.117,
"duration_us": {
"n": 1,
"p50": 14340331.117,
"p95": 14340331.117,
"mean": 14340331.117,
"min": 14340331.117,
"max": 14340331.117
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 14055462.854,
"duration_us": {
"n": 117,
"p50": 100917.275,
"p95": 267812.64,
"mean": 120132.16114529915,
"min": 91455.588,
"max": 418784.352
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_1(1)",
"duration_sum_us": 12497387.908,
"duration_us": {
"n": 117,
"p50": 95778.929,
"p95": 179187.172,
"mean": 106815.28126495726,
"min": 12792.463,
"max": 384924.79
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_64x2_tn_align8>(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_64x2_tn_align8::Params)",
"duration_sum_us": 6392248.861,
"duration_us": {
"n": 26577,
"p50": 152.645,
"p95": 367.752,
"mean": 240.51807431237535,
"min": 95.397,
"max": 12182.814
}
},
{
"category": "kernel",
"name": "std::enable_if<!(false), void>::type internal::gemvx::kernel<int, int, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, float, false, true, true, false, 6, false, cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float> >(cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16 const>, cublasGemvTensorStridedBatched<__nv_bfloat16>, float>)",
"duration_sum_us": 3167439.214,
"duration_us": {
"n": 14979,
"p50": 159.364,
"p95": 292.263,
"mean": 211.45865638560653,
"min": 96.13,
"max": 10834.725
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 3142457.444,
"duration_us": {
"n": 4893,
"p50": 12.694,
"p95": 95.722,
"mean": 642.2353247496424,
"min": 0.16,
"max": 107938.411
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 3072296.806,
"duration_us": {
"n": 2652,
"p50": 1.944,
"p95": 132.732,
"mean": 1158.4829585218702,
"min": 0.02,
"max": 107952.829
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 3069445.812,
"duration_us": {
"n": 1155,
"p50": 24.827,
"p95": 24551.58,
"mean": 2657.5288415584414,
"min": 1.613,
"max": 107951.406
}
},
{
"category": "cuda_runtime",
"name": "cudaStreamSynchronize",
"duration_sum_us": 3036074.002,
"duration_us": {
"n": 141,
"p50": 24015.406,
"p95": 30940.217,
"mean": 21532.43973049645,
"min": 1.323,
"max": 107915.608
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_128x2_tn_align8>(cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_128x2_tn_align8::Params)",
"duration_sum_us": 2226137.006,
"duration_us": {
"n": 8859,
"p50": 198.95,
"p95": 302.278,
"mean": 251.28536019866803,
"min": 187.269,
"max": 7693.401
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_0(0)",
"duration_sum_us": 724447.939,
"duration_us": {
"n": 3,
"p50": 249292.029,
"p95": 249473.366,
"mean": 241482.64633333334,
"min": 225682.544,
"max": 249473.366
}
},
{
"category": "overhead",
"name": "Activity Buffer Request",
"duration_sum_us": 394710.156,
"duration_us": {
"n": 133,
"p50": 2971.443,
"p95": 3170.29,
"mean": 2967.7455338345867,
"min": 1721.78,
"max": 3686.167
}
},
{
"category": "kernel",
"name": "void pytorch_flash::flash_fwd_kernel<Flash_fwd_kernel_traits<128, 128, 64, 4, false, false, cutlass::bfloat16_t, Flash_kernel_traits<128, 128, 64, 4, cutlass::bfloat16_t> >, false, true, false, false, false, true, false, false>(pytorch_flash::Flash_fwd_params)",
"duration_sum_us": 352973.4,
"duration_us": {
"n": 8775,
"p50": 24.77,
"p95": 72.804,
"mean": 40.22488888888889,
"min": 10.145,
"max": 4100.907
}
},
{
"category": "cuda_runtime",
"name": "cudaGraphLaunch",
"duration_sum_us": 311104.506,
"duration_us": {
"n": 234,
"p50": 1349.566,
"p95": 2220.347,
"mean": 1329.506435897436,
"min": 461.747,
"max": 5027.754
}
},
{
"category": "overhead",
"name": "Buffer Flush",
"duration_sum_us": 198951.703,
"duration_us": {
"n": 199,
"p50": 938.933,
"p95": 2006.544,
"mean": 999.7573015075377,
"min": 90.973,
"max": 22321.617
}
},
{
"category": "kernel",
"name": "void cublasLt::splitKreduce_kernel<32, 16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, false, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, true, false, false, false>(cublasLt::cublasSplitKParams<float>, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, float const*, float const*, __nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, void*, long, float*, int*, float*, float*, float const*, float const*, float const*, float const*, float const*)",
"duration_sum_us": 149593.069,
"duration_us": {
"n": 17718,
"p50": 5.504,
"p95": 11.649,
"mean": 8.442999717801106,
"min": 1.824,
"max": 4885.485
}
},
{
"category": "cuda_runtime",
"name": "cudaLaunchKernel",
"duration_sum_us": 142610.394,
"duration_us": {
"n": 5982,
"p50": 8.796,
"p95": 43.974,
"mean": 23.839918756268805,
"min": 3.957,
"max": 1121.88
}
},
{
"category": "kernel",
"name": "void flash::flash_fwd_splitkv_kernel<Flash_fwd_kernel_traits<128, 64, 128, 4, false, false, cutlass::bfloat16_t, Flash_kernel_traits<128, 64, 128, 4, cutlass::bfloat16_t> >, false, false, false, false, true, false, true, false>(flash::Flash_fwd_params)",
"duration_sum_us": 136441.255,
"duration_us": {
"n": 3276,
"p50": 26.209,
"p95": 78.05,
"mean": 41.64873473748474,
"min": 13.184,
"max": 4494.261
}
},
{
"category": "kernel",
"name": "triton_per_fused__scaled_dot_product_flash_attention__to_copy__unsafe_view_add_cat_clone_index_mean_mul_neg_pow_rsqrt_slice_split_transpose_unsqueeze_view_1",
"duration_sum_us": 114037.38,
"duration_us": {
"n": 8775,
"p50": 9.697,
"p95": 14.816,
"mean": 12.995712820512821,
"min": 4.448,
"max": 3574.43
}
},
{
"category": "kernel",
"name": "void at::native::sbtopk::gatherTopK<c10::BFloat16, unsigned int, 1, false>(at::cuda::detail::TensorInfo<c10::BFloat16 const, unsigned int>, unsigned int, unsigned int, bool, unsigned int, unsigned int, at::cuda::detail::TensorInfo<c10::BFloat16, unsigned int>, unsigned int, at::cuda::detail::TensorInfo<long, unsigned int>, unsigned int, c10::BFloat16*)",
"duration_sum_us": 97492.728,
"duration_us": {
"n": 1755,
"p50": 32.032,
"p95": 107.174,
"mean": 55.551411965811965,
"min": 14.4,
"max": 3202.004
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(19)_generation_0(0)",
"duration_sum_us": 97089.95999999999,
"duration_us": {
"n": 3,
"p50": 30755.484,
"p95": 36347.162,
"mean": 32363.319999999996,
"min": 29987.314,
"max": 36347.162
}
},
{
"category": "kernel",
"name": "void flash::flash_fwd_splitkv_combine_kernel<Flash_fwd_kernel_traits<128, 64, 128, 4, false, false, cutlass::bfloat16_t, Flash_kernel_traits<128, 64, 128, 4, cutlass::bfloat16_t> >, 4, 4, true>(flash::Flash_fwd_params)",
"duration_sum_us": 87114.529,
"duration_us": {
"n": 3276,
"p50": 16.481,
"p95": 48.577,
"mean": 26.59173656898657,
"min": 8.0,
"max": 3448.666
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 82164.028,
"duration_us": {
"n": 4356,
"p50": 10.19,
"p95": 41.048,
"mean": 18.862265381083564,
"min": 4.028,
"max": 2519.668
}
},
{
"category": "cpu_op",
"name": "aten::index_copy_",
"duration_sum_us": 80316.216,
"duration_us": {
"n": 234,
"p50": 256.15,
"p95": 1040.26,
"mean": 343.2316923076923,
"min": 14.187,
"max": 1180.462
}
},
{
"category": "kernel",
"name": "triton_per_fused__scaled_dot_product_flash_attention__to_copy__unsafe_view_add_cat_clone_index_mean_mul_neg_pow_rsqrt_slice_split_transpose_unsqueeze_view_2",
"duration_sum_us": 62208.374,
"duration_us": {
"n": 8775,
"p50": 6.208,
"p95": 10.368,
"mean": 7.089273390313391,
"min": 1.824,
"max": 2015.093
}
},
{
"category": "user_annotation",
"name": "execute_context_1(19)_generation_0(0)",
"duration_sum_us": 60235.267,
"duration_us": {
"n": 3,
"p50": 20541.336,
"p95": 21720.068,
"mean": 20078.422333333332,
"min": 17973.863,
"max": 21720.068
}
},
{
"category": "kernel",
"name": "void at::native::elementwise_kernel<128, 4, at::native::gpu_kernel_impl_nocast<at::native::(anonymous namespace)::CompareFunctor<c10::BFloat16> >(at::TensorIteratorBase&, at::native::(anonymous namespace)::CompareFunctor<c10::BFloat16> const&)::{lambda(int)#1}>(int, at::native::gpu_kernel_impl_nocast<at::native::(anonymous namespace)::CompareFunctor<c10::BFloat16> >(at::TensorIteratorBase&, at::native::(anonymous namespace)::CompareFunctor<c10::BFloat16> const&)::{lambda(int)#1})",
"duration_sum_us": 55480.076,
"duration_us": {
"n": 1755,
"p50": 23.936,
"p95": 33.859,
"mean": 31.61257891737892,
"min": 18.144,
"max": 3179.827
}
},
{
"category": "kernel",
"name": "triton_poi_fused__unsafe_view_mul_silu_split_with_sizes_5",
"duration_sum_us": 47016.378,
"duration_us": {
"n": 8775,
"p50": 4.065,
"p95": 7.168,
"mean": 5.357991794871794,
"min": 1.312,
"max": 2638.074
}
},
{
"category": "kernel",
"name": "memcpy32_post",
"duration_sum_us": 45956.95,
"duration_us": {
"n": 8775,
"p50": 4.0,
"p95": 6.816,
"mean": 5.237259259259259,
"min": 1.152,
"max": 3425.132
}
},
{
"category": "kernel",
"name": "void at::native::warpMergeSortKVInPlace<-2, -1, 128, 16, c10::BFloat16, long, at::native::GTOp<c10::BFloat16, true>, unsigned int, 32>(at::cuda::detail::TensorInfo<c10::BFloat16, unsigned int>, unsigned int, unsigned int, unsigned int, at::cuda::detail::TensorInfo<long, unsigned int>, unsigned int, at::native::GTOp<c10::BFloat16, true>, c10::BFloat16)",
"duration_sum_us": 43421.944,
"duration_us": {
"n": 1755,
"p50": 17.536,
"p95": 47.649,
"mean": 24.741848433048435,
"min": 8.032,
"max": 2306.079
}
},
{
"category": "kernel",
"name": "triton_per_fused__to_copy__unsafe_view_add_mean_mul_pow_rsqrt_4",
"duration_sum_us": 42233.815,
"duration_us": {
"n": 5265,
"p50": 7.2,
"p95": 11.617,
"mean": 8.021617283950619,
"min": 1.631,
"max": 1653.274
}
},
{
"category": "kernel",
"name": "triton_poi_fused__scaled_dot_product_flash_attention__to_copy__unsafe_view_add_clone_index_mean_mul_pow_rsqrt_slice_split_transpose_unsqueeze_view_3",
"duration_sum_us": 42198.534,
"duration_us": {
"n": 8775,
"p50": 3.68,
"p95": 6.145,
"mean": 4.808949743589744,
"min": 1.119,
"max": 3349.528
}
},
{
"category": "kernel",
"name": "void at::native::reduce_kernel<512, 1, at::native::ReduceOp<float, at::native::ArgMaxOps<float>, unsigned int, long, 4, 4> >(at::native::ReduceOp<float, at::native::ArgMaxOps<float>, unsigned int, long, 4, 4>)",
"duration_sum_us": 42119.386,
"duration_us": {
"n": 1875,
"p50": 16.864,
"p95": 28.449,
"mean": 22.463672533333334,
"min": 4.32,
"max": 2354.978
}
},
{
"category": "kernel",
"name": "void vllm::reshape_and_cache_flash_kernel<__nv_bfloat16, __nv_bfloat16, (vllm::Fp8KVCacheDataType)0>(__nv_bfloat16 const*, __nv_bfloat16 const*, __nv_bfloat16*, __nv_bfloat16*, long const*, long, long, long, long, long, int, int, int, float const*, float const*, int)",
"duration_sum_us": 32087.463,
"duration_us": {
"n": 3360,
"p50": 6.432,
"p95": 15.265,
"mean": 9.54984017857143,
"min": 2.464,
"max": 3980.296
}
},
{
"category": "kernel",
"name": "triton_per_fused_fused_add_rms_norm_2",
"duration_sum_us": 31864.192,
"duration_us": {
"n": 3360,
"p50": 7.648,
"p95": 12.192,
"mean": 9.483390476190475,
"min": 3.871,
"max": 3390.296
}
},
{
"category": "kernel",
"name": "triton_per_fused__to_copy__unsafe_view_add_mean_mul_pow_rsqrt_6",
"duration_sum_us": 29685.574,
"duration_us": {
"n": 3510,
"p50": 7.904,
"p95": 12.416,
"mean": 8.45742849002849,
"min": 1.888,
"max": 1123.293
}
},
{
"category": "kernel",
"name": "triton_per_fused__to_copy__unsafe_view_add_mean_mul_pow_rsqrt_7",
"duration_sum_us": 29417.597,
"duration_us": {
"n": 3510,
"p50": 8.193,
"p95": 12.705,
"mean": 8.381081766381767,
"min": 1.696,
"max": 580.172
}
},
{
"category": "kernel",
"name": "triton_per_fused_fused_add_rms_norm_0",
"duration_sum_us": 29163.093,
"duration_us": {
"n": 3360,
"p50": 7.328,
"p95": 11.136,
"mean": 8.679491964285715,
"min": 3.872,
"max": 1878.993
}
},
{
"category": "cpu_op",
"name": "aten::fill_",
"duration_sum_us": 27803.471,
"duration_us": {
"n": 1566,
"p50": 8.286,
"p95": 64.593,
"mean": 17.75445146871009,
"min": 0.591,
"max": 3166.888
}
}
]
},
{
"source": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\analysis\\experiments\\e2e_profiling_20260922\\tts-cuda-wsl-trace\\traces\\20260922-185614_e2e-stage1\\trace_rank0.json",
"scope": "one process trace; event duration sums overlap and are not E2E time",
"categories": {
"user_annotation": {
"events": 8790,
"duration_sum_us": 900577.0068359375,
"busy_union_us": 900577.0068359375
},
"cpu_op": {
"events": 4851,
"duration_sum_us": 8291738.6904296875,
"busy_union_us": 2772566.900390625
},
"overhead": {
"events": 4,
"duration_sum_us": 6492.2646484375,
"busy_union_us": 6492.2646484375
},
"cuda_runtime": {
"events": 27118,
"duration_sum_us": 2856002.494140625,
"busy_union_us": 2856002.494140625
},
"gpu_memcpy": {
"events": 243,
"duration_sum_us": 1340.9912109375,
"busy_union_us": 1340.9912109375
},
"kernel": {
"events": 10977,
"duration_sum_us": 2628357.83203125,
"busy_union_us": 2628302.375
},
"gpu_memset": {
"events": 48,
"duration_sum_us": 83.77734375,
"busy_union_us": 83.77734375
},
"gpu_user_annotation": {
"events": 15,
"duration_sum_us": 2796922.3486328125,
"busy_union_us": 2796922.3486328125
},
"Trace": {
"events": 1,
"duration_sum_us": 27790743.123046875,
"busy_union_us": 27790743.123046875
}
},
"largest_ops_by_duration_sum": [
{
"category": "Trace",
"name": "PyTorch Profiler (0)",
"duration_sum_us": 27790743.123,
"duration_us": {
"n": 1,
"p50": 27790743.123,
"p95": 27790743.123,
"mean": 27790743.123,
"min": 27790743.123,
"max": 27790743.123
}
},
{
"category": "cpu_op",
"name": "aten::copy_",
"duration_sum_us": 2762516.567,
"duration_us": {
"n": 666,
"p50": 1.072,
"p95": 56.868,
"mean": 4147.922773273273,
"min": 0.16,
"max": 417423.016
}
},
{
"category": "cuda_runtime",
"name": "cudaMemcpyAsync",
"duration_sum_us": 2759463.916,
"duration_us": {
"n": 243,
"p50": 11.582,
"p95": 77868.615,
"mean": 11355.818584362141,
"min": 4.759,
"max": 417384.833
}
},
{
"category": "cpu_op",
"name": "aten::to",
"duration_sum_us": 2758598.719,
"duration_us": {
"n": 165,
"p50": 8.476,
"p95": 112690.643,
"mean": 16718.780115151516,
"min": 0.05,
"max": 417447.392
}
},
{
"category": "cpu_op",
"name": "aten::_to_copy",
"duration_sum_us": 2758072.923,
"duration_us": {
"n": 75,
"p50": 28.283,
"p95": 299115.061,
"mean": 36774.30564,
"min": 7.925,
"max": 417443.996
}
},
{
"category": "user_annotation",
"name": "execute_context_0(0)_generation_0(0)",
"duration_sum_us": 848333.409,
"duration_us": {
"n": 8775,
"p50": 60.275,
"p95": 261.426,
"mean": 96.67617196581196,
"min": 47.31,
"max": 704.25
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(256)_generation_0(0)",
"duration_sum_us": 836616.084,
"duration_us": {
"n": 3,
"p50": 239768.145,
"p95": 362783.999,
"mean": 278872.028,
"min": 234063.94,
"max": 362783.999
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(128)_generation_0(0)",
"duration_sum_us": 717274.3150000001,
"duration_us": {
"n": 4,
"p50": 141171.473,
"p95": 313652.59,
"mean": 179318.57875000002,
"min": 110614.585,
"max": 313652.59
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(112)_generation_0(0)",
"duration_sum_us": 419421.56,
"duration_us": {
"n": 1,
"p50": 419421.56,
"p95": 419421.56,
"mean": 419421.56,
"min": 419421.56,
"max": 419421.56
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(144)_generation_0(0)",
"duration_sum_us": 302326.497,
"duration_us": {
"n": 1,
"p50": 302326.497,
"p95": 302326.497,
"mean": 302326.497,
"min": 302326.497,
"max": 302326.497
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(64)_generation_0(0)",
"duration_sum_us": 288010.704,
"duration_us": {
"n": 3,
"p50": 93173.425,
"p95": 114662.991,
"mean": 96003.56800000001,
"min": 80174.288,
"max": 114662.991
}
},
{
"category": "kernel",
"name": "void cudnn::engines_precompiled::nchwToNhwcKernel<float, float, float, false, true, (cudnnKernelDataType_t)2>(cudnn::engines_precompiled::nchw2nhwc_params_t<float>, float const*, float*)",
"duration_sum_us": 238826.888,
"duration_us": {
"n": 336,
"p50": 68.641,
"p95": 4838.33,
"mean": 710.7943095238095,
"min": 4.352,
"max": 8910.789
}
},
{
"category": "kernel",
"name": "void implicit_convolve_sgemm<float, float, 128, 5, 5, 3, 3, 3, 1, false, false, false>(int, int, int, float const*, int, float*, float const*, kernel_conv_params, unsigned long long, int, float, float, int, float const*, float const*, bool, bool, int, int)",
"duration_sum_us": 236406.345,
"duration_us": {
"n": 30,
"p50": 4409.635,
"p95": 31316.0,
"mean": 7880.2115,
"min": 412.553,
"max": 32261.685
}
},
{
"category": "gpu_user_annotation",
"name": "execute_context_1(32)_generation_0(0)",
"duration_sum_us": 233273.19,
"duration_us": {
"n": 3,
"p50": 68905.811,
"p95": 99068.406,
"mean": 77757.73,
"min": 65298.973,
"max": 99068.406
}
},
{
"category": "kernel",
"name": "void cutlass__5x_cudnn::Kernel<cutlass_tensorop_s1688fprop_optimized_tf32_64x64_32x5_nhwc_align4>(cutlass_tensorop_s1688fprop_optimized_tf32_64x64_32x5_nhwc_align4::Params)",
"duration_sum_us": 230859.55,
"duration_us": {
"n": 120,
"p50": 623.373,
"p95": 6159.334,
"mean": 1923.8295833333332,
"min": 33.121,
"max": 9649.969
}
},
{
"category": "kernel",
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::CUDAFunctor_add<float>, std::array<char*, 3ul> >(int, at::native::CUDAFunctor_add<float>, std::array<char*, 3ul>)",
"duration_sum_us": 199130.316,
"duration_us": {
"n": 645,
"p50": 5.248,
"p95": 1570.754,
"mean": 308.72917209302324,
"min": 1.568,
"max": 10724.916
}
},
{
"category": "kernel",
"name": "void precomputed_convolve_sgemm<float, 1024, 5, 5, 4, 3, 3, 1, false>(int, int, int, float const*, int, float*, float const*, kernel_conv_params, unsigned long long, int, float, float, int, bool, bool, float const*, float const*, int*)",
"duration_sum_us": 176016.055,
"duration_us": {
"n": 33,
"p50": 5349.844,
"p95": 10881.652,
"mean": 5333.819848484848,
"min": 547.436,
"max": 11827.152
}
},
{
"category": "kernel",
"name": "void cudnn::detail::dgrad_engine<float, 512, 6, 5, 3, 3, 3, false>(int, int, int, float const*, int, float const*, int, float*, kernel_grad_params, unsigned long long, int, unsigned long long, int, float, int, int, int)",
"duration_sum_us": 158179.096,
"duration_us": {
"n": 45,
"p50": 1506.886,
"p95": 12525.306,
"mean": 3515.091022222222,
"min": 94.947,
"max": 15997.396
}
},
{
"category": "kernel",
"name": "_kernel",
"duration_sum_us": 151273.359,
"duration_us": {
"n": 435,
"p50": 21.088,
"p95": 3003.822,
"mean": 347.75484827586206,
"min": 4.64,
"max": 8295.861
}
},
{
"category": "kernel",
"name": "void at::native::elementwise_kernel<128, 2, at::native::gpu_kernel_impl_nocast<at::native::CUDAFunctor_add<float> >(at::TensorIteratorBase&, at::native::CUDAFunctor_add<float> const&)::{lambda(int)#1}>(int, at::native::gpu_kernel_impl_nocast<at::native::CUDAFunctor_add<float> >(at::TensorIteratorBase&, at::native::CUDAFunctor_add<float> const&)::{lambda(int)#1})",
"duration_sum_us": 132090.664,
"duration_us": {
"n": 780,
"p50": 11.04,
"p95": 476.778,
"mean": 169.34700512820513,
"min": 1.92,
"max": 5933.829
}
},
{
"category": "kernel",
"name": "void implicit_convolve_sgemm<float, float, 128, 5, 5, 3, 3, 3, 1, false, false, true>(int, int, int, float const*, int, float*, float const*, kernel_conv_params, unsigned long long, int, float, float, int, float const*, float const*, bool, bool, int, int)",
"duration_sum_us": 127146.523,
"duration_us": {
"n": 30,
"p50": 749.745,
"p95": 23634.964,
"mean": 4238.217433333333,
"min": 188.421,
"max": 28554.306
}
},
{
"category": "kernel",
"name": "void at::native::elementwise_kernel<128, 2, at::native::gpu_kernel_impl_nocast<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}>(at::TensorIteratorBase&, at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1} const&)::{lambda(int)#1}>(int, at::native::gpu_kernel_impl_nocast<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1}>(at::TensorIteratorBase&, at::native::direct_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda()#3}::operator()() const::{lambda()#7}::operator()() const::{lambda(float)#1} const&)::{lambda(int)#1})",
"duration_sum_us": 126594.793,
"duration_us": {
"n": 549,
"p50": 7.233,
"p95": 571.245,
"mean": 230.5916083788707,
"min": 1.888,
"max": 9393.971
}
},
{
"category": "kernel",
"name": "memcpy128",
"duration_sum_us": 80458.009,
"duration_us": {
"n": 144,
"p50": 14.784,
"p95": 3849.619,
"mean": 558.7361736111111,
"min": 3.841,
"max": 8203.442
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_tensorop_s1688gemm_256x128_16x3_nn_align4>(cutlass_80_tensorop_s1688gemm_256x128_16x3_nn_align4::Params)",
"duration_sum_us": 78652.89199999999,
"duration_us": {
"n": 54,
"p50": 210.629,
"p95": 7044.857,
"mean": 1456.535037037037,
"min": 25.44,
"max": 8099.088
}
},
{
"category": "kernel",
"name": "sm80_xmma_dgrad_implicit_gemm_indexed_tf32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x16_stage4_warpsize2x2x1_g1_tensor16x8x8_strided_execute_kernel__5x_cudnn",
"duration_sum_us": 75293.188,
"duration_us": {
"n": 9,
"p50": 9037.155,
"p95": 13082.46,
"mean": 8365.909777777777,
"min": 473.964,
"max": 13082.46
}
},
{
"category": "kernel",
"name": "void at::native::vectorized_elementwise_kernel<4, at::native::FillFunctor<float>, std::array<char*, 1ul> >(int, at::native::FillFunctor<float>, std::array<char*, 1ul>)",
"duration_sum_us": 62214.47,
"duration_us": {
"n": 627,
"p50": 3.968,
"p95": 282.856,
"mean": 99.22562998405104,
"min": 1.088,
"max": 7849.488
}
},
{
"category": "kernel",
"name": "void cudnn::engines_precompiled::nhwcToNchwKernel<float, float, float, true, false, (cudnnKernelDataType_t)0>(cudnn::engines_precompiled::nhwc2nchw_params_t<float>, float const*, float*)",
"duration_sum_us": 49203.447,
"duration_us": {
"n": 168,
"p50": 14.816,
"p95": 652.337,
"mean": 292.8776607142857,
"min": 2.624,
"max": 7753.448
}
},
{
"category": "kernel",
"name": "void sgemm_largek_lds64<true, false, 5, 5, 4, 4, 4, 34>(float*, float const*, float const*, int, int, int, int, int, int, float const*, float const*, float, float, int, int, int*, int*)",
"duration_sum_us": 43372.548,
"duration_us": {
"n": 24,
"p50": 602.669,
"p95": 4720.487,
"mean": 1807.1895000000002,
"min": 68.225,
"max": 6226.784
}
},
{
"category": "kernel",
"name": "void gemmSN_TN_kernel<float, 128, 16, 2, 4, 6, 7, false, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float> >(cublasGemmSmallNParams<cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float>, float>)",
"duration_sum_us": 40096.021,
"duration_us": {
"n": 177,
"p50": 61.058,
"p95": 1245.723,
"mean": 226.53119209039548,
"min": 51.713,
"max": 6258.119
}
},
{
"category": "kernel",
"name": "void at::native::elementwise_kernel<128, 2, at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float> > >(at::TensorIteratorBase&, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float> > const&)::{lambda(int)#1}>(int, at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float> > >(at::TensorIteratorBase&, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float> > const&)::{lambda(int)#1})",
"duration_sum_us": 38187.497,
"duration_us": {
"n": 240,
"p50": 59.202,
"p95": 81.474,
"mean": 159.11457083333335,
"min": 52.001,
"max": 5432.041
}
},
{
"category": "cuda_runtime",
"name": "cudaEventSynchronize",
"duration_sum_us": 35539.829,
"duration_us": {
"n": 8805,
"p50": 3.326,
"p95": 6.993,
"mean": 4.036323566155593,
"min": 0.601,
"max": 177.256
}
},
{
"category": "kernel",
"name": "void gemmSN_TN_kernel<float, 128, 16, 2, 4, 14, 15, false, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float> >(cublasGemmSmallNParams<cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float>, float>)",
"duration_sum_us": 34292.937,
"duration_us": {
"n": 171,
"p50": 68.386,
"p95": 1008.503,
"mean": 200.54349122807017,
"min": 55.234,
"max": 4429.251
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_simt_sgemm_128x64_8x5_tn_align1>(cutlass_80_simt_sgemm_128x64_8x5_tn_align1::Params)",
"duration_sum_us": 33901.108,
"duration_us": {
"n": 219,
"p50": 64.386,
"p95": 554.475,
"mean": 154.7995799086758,
"min": 53.729,
"max": 2830.216
}
},
{
"category": "kernel",
"name": "void cutlass__5x_cudnn::Kernel<cutlass_tensorop_s1688dgrad_optimized_tf32_128x64_32x3_nhwc_align4>(cutlass_tensorop_s1688dgrad_optimized_tf32_128x64_32x3_nhwc_align4::Params)",
"duration_sum_us": 32652.173,
"duration_us": {
"n": 15,
"p50": 618.32,
"p95": 6880.212,
"mean": 2176.811533333333,
"min": 188.005,
"max": 6880.212
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_tensorop_s1688gemm_256x64_16x4_nn_align4>(cutlass_80_tensorop_s1688gemm_256x64_16x4_nn_align4::Params)",
"duration_sum_us": 31139.533,
"duration_us": {
"n": 45,
"p50": 123.843,
"p95": 4069.915,
"mean": 691.9896222222222,
"min": 26.208,
"max": 8983.53
}
},
{
"category": "kernel",
"name": "void gemmSN_TN_kernel<float, 128, 16, 2, 4, 2, 2, true, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float> >(cublasGemmSmallNParams<cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float const>, cublasGemvTensorStridedBatched<float>, float>)",
"duration_sum_us": 28615.551,
"duration_us": {
"n": 171,
"p50": 60.833,
"p95": 1178.553,
"mean": 167.34240350877192,
"min": 51.649,
"max": 2831.145
}
},
{
"category": "cuda_runtime",
"name": "cudaEventRecordWithFlags",
"duration_sum_us": 24732.206,
"duration_us": {
"n": 8820,
"p50": 2.324,
"p95": 5.45,
"mean": 2.8041049886621314,
"min": 0.761,
"max": 40.457
}
},
{
"category": "kernel",
"name": "void cutlass::Kernel2<cutlass_80_simt_sgemm_128x32_8x5_tn_align1>(cutlass_80_simt_sgemm_128x32_8x5_tn_align1::Params)",
"duration_sum_us": 22314.802,
"duration_us": {
"n": 120,
"p50": 66.465,
"p95": 85.762,
"mean": 185.95668333333333,
"min": 53.377,
"max": 4999.052
}
},
{
"category": "cuda_runtime",
"name": "cudaGraphLaunch",
"duration_sum_us": 19267.319,
"duration_us": {
"n": 15,
"p50": 1049.524,
"p95": 4183.104,
"mean": 1284.4879333333333,
"min": 980.502,
"max": 4183.104
}
},
{
"category": "kernel",
"name": "void at::native::elementwise_kernel<128, 2, at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float> > >(at::TensorIteratorBase&, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float> > const&)::{lambda(int)#1}>(int, at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float> > >(at::TensorIteratorBase&, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float> > const&)::{lambda(int)#1})",
"duration_sum_us": 16596.245,
"duration_us": {
"n": 1260,
"p50": 7.137,
"p95": 11.456,
"mean": 13.171623015873015,
"min": 1.663,
"max": 2372.115
}
}
]
}
],
"trace_analysis": "requested"
}