[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9e2117f5e
commit
86fcfe95f1
23 files changed
+1957
-200
No files matched your search
@@ -115,11 +115,27 @@ def allocate_kv_pool_with_views(
|
||||
cache_shape = (num_blocks, block_size, num_kv_heads, head_dim)
|
||||
flat_shape = (num_blocks * block_size, num_kv_heads, head_dim)
|
||||
for _ in range(num_layers):
|
||||
k = torch.empty(cache_shape, dtype=dtype, device=device)
|
||||
v = torch.empty(cache_shape, dtype=dtype, device=device)
|
||||
kv_pools.append([k, v])
|
||||
k_pools.append(k.reshape(flat_shape))
|
||||
v_pools.append(v.reshape(flat_shape))
|
||||
# The flat slot views are what the paged-write custom op mutates inside the
|
||||
# compiled block graphs, so they must be the *base* allocations: dynamo only
|
||||
# honours mark_static_address on the tensor object it sees as a graph input,
|
||||
# and a view of a static base is still classified as a mutated input, which
|
||||
# makes inductor skip CUDA graphs for every block ("skipping cudagraphs due
|
||||
# to mutated inputs"). The block-table layout is therefore the view here.
|
||||
k = torch.empty(flat_shape, dtype=dtype, device=device)
|
||||
v = torch.empty(flat_shape, dtype=dtype, device=device)
|
||||
for pool in (k, v):
|
||||
# Block 0 is the manager's null block: block tables are tail-padded with
|
||||
# it and it is never written. The contiguous-K/V gather path may read
|
||||
# (masked) rows from it, so it must hold finite values; every other
|
||||
# block is fully written before it is read.
|
||||
pool[:block_size].zero_()
|
||||
# Session-lifetime storage: tell dynamo/inductor the address is static so
|
||||
# CUDA-graph trees (mode="reduce-overhead") mutate it in place instead of
|
||||
# copying ~300 MB per layer per replay (or skipping the graph).
|
||||
torch._dynamo.mark_static_address(pool)
|
||||
kv_pools.append([k.view(cache_shape), v.view(cache_shape)])
|
||||
k_pools.append(k)
|
||||
v_pools.append(v)
|
||||
return kv_pools, k_pools, v_pools
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user