[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Fable 5.1 authored and GitHub committed 2026-09-21 06:12:45 +00:00
1 parent a9e2117f5e
commit 86fcfe95f1
23 files changed
+1957 -200

No files matched your search

@@ -115,11 +115,27 @@ def allocate_kv_pool_with_views(
cache_shape = (num_blocks, block_size, num_kv_heads, head_dim)
flat_shape = (num_blocks * block_size, num_kv_heads, head_dim)
for _ in range(num_layers):
k = torch.empty(cache_shape, dtype=dtype, device=device)
v = torch.empty(cache_shape, dtype=dtype, device=device)
kv_pools.append([k, v])
k_pools.append(k.reshape(flat_shape))
v_pools.append(v.reshape(flat_shape))
# The flat slot views are what the paged-write custom op mutates inside the
# compiled block graphs, so they must be the *base* allocations: dynamo only
# honours mark_static_address on the tensor object it sees as a graph input,
# and a view of a static base is still classified as a mutated input, which
# makes inductor skip CUDA graphs for every block ("skipping cudagraphs due
# to mutated inputs"). The block-table layout is therefore the view here.
k = torch.empty(flat_shape, dtype=dtype, device=device)
v = torch.empty(flat_shape, dtype=dtype, device=device)
for pool in (k, v):
# Block 0 is the manager's null block: block tables are tail-padded with
# it and it is never written. The contiguous-K/V gather path may read
# (masked) rows from it, so it must hold finite values; every other
# block is fully written before it is read.
pool[:block_size].zero_()
# Session-lifetime storage: tell dynamo/inductor the address is static so
# CUDA-graph trees (mode="reduce-overhead") mutate it in place instead of
# copying ~300 MB per layer per replay (or skipping the graph).
torch._dynamo.mark_static_address(pool)
kv_pools.append([k.view(cache_shape), v.view(cache_shape)])
k_pools.append(k)
v_pools.append(v)
return kv_pools, k_pools, v_pools