From 289c13df683f3799fcb6a11dcbe5cd6d2839fcb2 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 14 Sep 2026 13:19:36 +0800 Subject: [PATCH] cpu_int4: tune the repack chunk to 2048 rows from a measured sweep 512 was a first guess. Repacking the 131072-row tied embedding at 24 threads: rows chunks time int32 transient whole 1 208 ms 1024 MiB 512 256 116 ms 4 MiB 2048 64 79 ms 16 MiB 8192 16 89 ms 64 MiB 32768 4 129 ms 256 MiB 2048 is both the fastest and nearly the leanest. The sweep also settles a suspicion: chunking is faster than packing the whole matrix, because the working set stays in cache, so this costs nothing at load. The one pass that appeared to show a 30 s init regression was the pass the harness had flagged as contended. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01L1ax5wYLA6XbqEL1i8L9gZ --- .../layers/quantization/cpu_int4.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/vllm_omni/model_executor/layers/quantization/cpu_int4.py b/vllm_omni/model_executor/layers/quantization/cpu_int4.py index 39caa2cf9..d943d398b 100644 --- a/vllm_omni/model_executor/layers/quantization/cpu_int4.py +++ b/vllm_omni/model_executor/layers/quantization/cpu_int4.py @@ -155,9 +155,23 @@ def quantize_groupwise(w: torch.Tensor, group: int): return pack_nibbles(q.reshape(n, k)), scale, zero -REPACK_ROWS = 512 +REPACK_ROWS = 2048 """Output channels repacked at a time, to bound the load-time peak. +2048 because it is both the fastest and nearly the leanest. Repacking the +131072-row tied embedding, 24 threads: + + rows chunks time int32 transient + whole 1 208 ms 1024 MiB + 512 256 116 ms 4 MiB + 2048 64 79 ms 16 MiB + 8192 16 89 ms 64 MiB + 32768 4 129 ms 256 MiB + +Chunking is *faster* than packing the whole matrix -- the working set stays in +cache -- so this costs nothing at load; the earlier suspicion that it cost +30 s of init came from a pass the harness had flagged as contended. + ``unpack_nibbles`` expands 4-bit codes to int32 -- 4 bytes per weight, eight times the packed form -- and the kernel's packer wants int32. Doing that for a whole matrix makes the transient the largest thing in the process: 104 MiB for