[AR-Diffusion] Page in kernel-legal blocks so the default resolution can run (#6481)
Signed-off-by: linzhenpl07 <linzhenpl07@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
8 files changed
+1229
-50
No files matched your search
@@ -376,10 +376,17 @@ def test_lru_eviction_releases_blocks_and_notifies_pipeline():
|
||||
|
||||
def test_budget_reduced_capacity_drives_runner_lru():
|
||||
pipeline = CapablePipeline(tiny_spec(capacity=2))
|
||||
# One tiny LingBot-like session requires 128 bytes; two require 192.
|
||||
# One tiny LingBot-like session requires 1,808 bytes; two require 2,592.
|
||||
#
|
||||
# These read 128 and 192 while the paging unit was the frame. tiny_spec
|
||||
# declares one token per frame, so it used to page a single token at a
|
||||
# time -- a block size no attention kernel accepts. Paging at 16 makes
|
||||
# each page sixteen times larger. The block *counts* are unchanged, and
|
||||
# so is what this test is about: a budget that fits fewer sessions than
|
||||
# the pipeline asked for must drive the runner's LRU.
|
||||
runner = make_runner(
|
||||
pipeline,
|
||||
available_bytes=128,
|
||||
available_bytes=2048,
|
||||
gpu_memory_fraction=1.0,
|
||||
)
|
||||
kv = runner.kv_cache
|
||||
@@ -665,3 +672,106 @@ def test_pipeline_without_warmup_provider_is_safely_skipped(monkeypatch):
|
||||
monkeypatch.setattr(runner, "execute_model", fail_if_called)
|
||||
runner._warmup_ar_rollout()
|
||||
assert execute.called is False
|
||||
|
||||
|
||||
# ── paging unit vs eviction unit ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_frame_that_is_already_a_legal_block_keeps_its_paging():
|
||||
"""Every resolution that runs today must page exactly as it does today."""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
for tokens_per_frame in (16, 320, 1440, 4096):
|
||||
assert tokens_per_frame % 16 == 0
|
||||
assert paging_block_size(tokens_per_frame) == tokens_per_frame
|
||||
|
||||
|
||||
def test_a_frame_the_kernel_would_reject_pages_at_the_finest_legal_unit():
|
||||
"""832x480 is the checkpoint's own default and the kernel rejects it.
|
||||
|
||||
(480/16) x (832/16) = 1560 tokens per frame, and 1560 % 16 == 8, so a
|
||||
frame-sized block is not something FlashAttention's paged kernel accepts.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
assert 1560 % 16 == 8
|
||||
assert paging_block_size(1560) == 16
|
||||
# And whatever it returns is always something the kernel takes.
|
||||
for tokens_per_frame in range(1, 200):
|
||||
assert paging_block_size(tokens_per_frame) % 16 == 0
|
||||
|
||||
|
||||
# ── the kernel's constraint is data, not a constant ─────────────────────────
|
||||
|
||||
|
||||
def _multiple_of(base):
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
return MultipleOf(base)
|
||||
|
||||
|
||||
def test_the_dispatching_module_answers_in_vllms_own_vocabulary():
|
||||
"""AR-Diffusion calls flash_attn itself, so no backend can be asked.
|
||||
|
||||
The answer has to come from the module that picks the kernel, and it has to
|
||||
have the shape vLLM uses, so the policy that consumes it does not care
|
||||
where it came from.
|
||||
"""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import supported_kernel_block_sizes
|
||||
|
||||
advertised = supported_kernel_block_sizes()
|
||||
assert advertised, "a kernel that accepts nothing cannot be paged for"
|
||||
for entry in advertised:
|
||||
assert isinstance(entry, int | MultipleOf)
|
||||
|
||||
|
||||
def test_a_kernel_that_takes_one_fixed_size_gets_that_size():
|
||||
"""hpc_attn advertises [64] -- not a multiple, a single legal value.
|
||||
|
||||
A frame of 1560 tokens is not 64, and 1560 % 64 == 24, so the frame cannot
|
||||
be the block and the only thing left to page at is 64. Assuming multiples
|
||||
of 16 here would hand the kernel a block size it rejects.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
assert 1560 % 64 == 24
|
||||
assert paging_block_size(1560, [64]) == 64
|
||||
assert paging_block_size(64, [64]) == 64
|
||||
assert paging_block_size(128, [64]) == 64 # a legal multiple is still not the advertised size
|
||||
|
||||
|
||||
def test_a_kernel_with_large_pages_still_keeps_a_whole_frame_when_it_fits():
|
||||
"""FlashInfer offers 128-and-up pages on Blackwell and not on Hopper.
|
||||
|
||||
The same checkpoint on the two cards must therefore page differently, which
|
||||
is exactly what a constant cannot express.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
hopper = [_multiple_of(16)]
|
||||
blackwell = [_multiple_of(16), 128, 256]
|
||||
assert paging_block_size(1440, hopper) == 1440 # a legal multiple of 16
|
||||
assert paging_block_size(1440, blackwell) == 1440
|
||||
assert paging_block_size(1560, hopper) == 16
|
||||
assert paging_block_size(1560, blackwell) == 16 # finest legal unit, not the largest
|
||||
|
||||
|
||||
def test_whatever_comes_back_is_always_something_the_kernel_accepts():
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
for advertised, check in (
|
||||
([_multiple_of(16)], lambda n: n % 16 == 0),
|
||||
([64], lambda n: n == 64),
|
||||
([_multiple_of(32), 48], lambda n: n % 32 == 0 or n == 48),
|
||||
):
|
||||
for tokens_per_frame in range(1, 300):
|
||||
assert check(paging_block_size(tokens_per_frame, advertised))
|
||||
|
||||
|
||||
def test_a_kernel_that_advertises_nothing_is_an_error_not_a_guess():
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
with pytest.raises(ValueError, match="no legal block size"):
|
||||
paging_block_size(1560, [])
|
||||
@@ -625,6 +625,74 @@ def test_non_contiguous_branch_indices_rejected():
|
||||
)
|
||||
|
||||
|
||||
# --- eviction against the block table ---------------------------------------
|
||||
|
||||
|
||||
class _RecordingManager(ChunkWindowManager):
|
||||
"""Capture the block range eviction asks for, without a real block pool.
|
||||
|
||||
``remove_skipped_blocks`` reads three attributes and calls one method. A
|
||||
pool would add nothing: the bug under test is the conversion from the
|
||||
spec's units into the block table's, and that is entirely visible here.
|
||||
"""
|
||||
|
||||
def __init__(self, spec):
|
||||
self.kv_cache_spec = spec
|
||||
self.sliding_window = spec.sliding_window
|
||||
self.block_size = spec.block_size
|
||||
self.freed: tuple[int, int] | None = None
|
||||
|
||||
def _remove_blocks_in_range(self, request_id, first_block, last_block):
|
||||
self.freed = (first_block, last_block)
|
||||
|
||||
|
||||
def test_sink_is_preserved_when_a_frame_spans_many_blocks():
|
||||
# The shipped geometry: 832x480 is 1560 tokens per frame, 1560 % 16 == 8 is
|
||||
# not a legal block size, so paging falls back to 16 and a frame spans 98
|
||||
# blocks. Every other eviction test uses chunk_size == BLOCK, where a frame
|
||||
# is a block and a sink counted in frames indexes the block table correctly
|
||||
# by accident -- which is why this went unnoticed.
|
||||
chunk_size, sink_chunks, window_chunks = 1560, 9, 9
|
||||
spec = ChunkWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
sliding_window=window_chunks * chunk_size,
|
||||
chunk_size=chunk_size,
|
||||
window_chunks=window_chunks,
|
||||
sink_chunks=sink_chunks,
|
||||
)
|
||||
manager = _RecordingManager(spec)
|
||||
|
||||
# Far enough past sink + window that eviction has definitely started.
|
||||
manager.remove_skipped_blocks("req", total_computed_tokens=30 * chunk_size)
|
||||
assert manager.freed is not None
|
||||
first_freed, last_freed = manager.freed
|
||||
|
||||
# The assertion is on the token boundary rather than a block count, so it
|
||||
# stays meaningful if the paging unit changes again.
|
||||
sink_tokens = sink_chunks * chunk_size
|
||||
assert first_freed * spec.block_size >= sink_tokens, (
|
||||
f"eviction starts at token {first_freed * spec.block_size}, inside the {sink_tokens}-token sink"
|
||||
)
|
||||
# And it must not over-protect: at most one straddling block beyond the sink.
|
||||
assert (first_freed - 1) * spec.block_size < sink_tokens
|
||||
# Only wholly-skipped blocks are released.
|
||||
skipped = manager.get_num_skipped_tokens(30 * chunk_size)
|
||||
assert last_freed * spec.block_size <= sink_tokens + skipped
|
||||
|
||||
|
||||
def test_sink_conversion_is_identity_when_a_frame_is_a_block():
|
||||
# Every geometry that paged one frame per block must be untouched.
|
||||
spec = make_spec(chunk_size=BLOCK, window_chunks=2, sink_chunks=1)
|
||||
manager = _RecordingManager(spec)
|
||||
manager.remove_skipped_blocks("req", total_computed_tokens=8 * BLOCK)
|
||||
assert manager.freed is not None
|
||||
first_freed, _ = manager.freed
|
||||
assert first_freed == spec.sink_chunks
|
||||
|
||||
|
||||
def test_staged_window_reuse_matches_a_full_gather():
|
||||
"""Reusing the staged history must be byte-identical to re-gathering the whole window.
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm_omni.experimental.ar_diffusion.kv_cache import (
|
||||
)
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache import paged_attention as paged_attention_module
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.config import KV_GATHER_ENV
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.paged import ChunkWindowManager
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
@@ -36,10 +37,26 @@ NEG = "negative"
|
||||
|
||||
|
||||
def make_state(
|
||||
*, num_layers=1, window_chunks=2, dtype=torch.float32, device=torch.device("cpu"), reuse_history_staging=False
|
||||
*,
|
||||
num_layers=1,
|
||||
window_chunks=2,
|
||||
chunk_size=BLOCK,
|
||||
sink_chunks=0,
|
||||
reset_at_boundary=False,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
reuse_history_staging=False,
|
||||
):
|
||||
"""Build a cache. ``chunk_size`` defaults to the block size, but the two are
|
||||
independent -- the shipped 832x480 gives 1560 tokens per frame against
|
||||
16-token blocks, so a frame is 97.5 blocks."""
|
||||
cfg = ARDiffusionKVConfig(
|
||||
enable=True, chunk_size=BLOCK, window_chunks=window_chunks, reuse_history_staging=reuse_history_staging
|
||||
enable=True,
|
||||
chunk_size=chunk_size,
|
||||
window_chunks=window_chunks,
|
||||
sink_chunks=sink_chunks,
|
||||
reset_at_boundary=reset_at_boundary,
|
||||
reuse_history_staging=reuse_history_staging,
|
||||
)
|
||||
kv = ARDiffusionKVCache(
|
||||
cfg,
|
||||
@@ -109,11 +126,13 @@ def _commit_video_span(
|
||||
n_chunks: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
chunk_size: int = BLOCK,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
ctx = st.get_kv_caches(kv_branch, seq_len=n_chunks * BLOCK, commit_current=True)[0].forward_ctx
|
||||
span = n_chunks * chunk_size
|
||||
ctx = st.get_kv_caches(kv_branch, seq_len=span, commit_current=True)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
k = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
v = torch.randn(1, n_chunks * BLOCK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
k = torch.randn(1, span, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
v = torch.randn(1, span, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv._k_pools[0][ctx.current_video_slot_mapping] = k[0]
|
||||
kv._v_pools[0][ctx.current_video_slot_mapping] = v[0]
|
||||
st.commit_paged_context(kv_branch)
|
||||
@@ -547,6 +566,644 @@ def test_custom_op_compiles_fullgraph_without_recompile_on_value_change(monkeypa
|
||||
torch._dynamo.reset()
|
||||
|
||||
|
||||
# ── a frame that is not a whole number of blocks ────────────────────────────
|
||||
|
||||
# 24 tokens per chunk against 16-token blocks leaves 8 over, the same remainder
|
||||
# the shipped 832x480 produces (1560 % 16 == 8). Every fixture above holds
|
||||
# chunk_size == BLOCK, so nothing there can reach this path.
|
||||
RAGGED_CHUNK = 24
|
||||
|
||||
|
||||
@pytest.mark.parametrize("commit_current", [False, True])
|
||||
@pytest.mark.parametrize("action_len", [0, 3])
|
||||
def test_a_ragged_chunk_still_matches_the_dense_reference(commit_current, action_len):
|
||||
"""Attention reads a sequence, not a set of blocks.
|
||||
|
||||
With one committed chunk of 24 tokens the history stops 8 slots into its
|
||||
last block. The committing path writes straight after it and is fine. The
|
||||
scratch path used to restart at slot zero of a fresh region, which left
|
||||
those 8 slots unwritten -- and since the kernel consumes the block table as
|
||||
one contiguous run, it read them as if they were tokens and shifted every
|
||||
token after them by 8. Shapes stayed correct throughout, so only the values
|
||||
showed it.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cpu")
|
||||
dtype = torch.float32
|
||||
# window_chunks=2 with one chunk of history keeps the whole sequence
|
||||
# resident, so the window does not also round up here; that case is
|
||||
# test_a_ragged_window_keeps_the_block_table_a_fixed_shape below.
|
||||
kv, st = make_state(dtype=dtype, device=device, window_chunks=2, chunk_size=RAGGED_CHUNK)
|
||||
|
||||
history_k, history_v = _commit_video_span(
|
||||
kv, st, kv_branch=POS, n_chunks=1, dtype=dtype, device=device, chunk_size=RAGGED_CHUNK
|
||||
)
|
||||
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=commit_current)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
assert ctx.start_offset == RAGGED_CHUNK % BLOCK == 8
|
||||
|
||||
current_k = torch.randn(1, RAGGED_CHUNK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
current_v = torch.randn(1, RAGGED_CHUNK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv._k_pools[0][ctx.current_video_slot_mapping] = current_k[0]
|
||||
kv._v_pools[0][ctx.current_video_slot_mapping] = current_v[0]
|
||||
|
||||
action_k = action_v = None
|
||||
if action_len:
|
||||
ctx.ensure_action_slots(action_len, device)
|
||||
action_k = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
action_v = torch.randn(1, action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv._k_pools[0][ctx.action_slot_mapping] = action_k[0]
|
||||
kv._v_pools[0][ctx.action_slot_mapping] = action_v[0]
|
||||
|
||||
query = torch.randn(1, RAGGED_CHUNK + action_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
block_table, query_start_loc, seq_lens, max_query_len, max_seq_len = ctx.build_block_table(
|
||||
action_len=action_len, query_len=query.shape[1], device=device
|
||||
)
|
||||
paged = ar_diffusion_paged_attention(
|
||||
query,
|
||||
kv.key_cache(0),
|
||||
kv.value_cache(0),
|
||||
block_table=block_table,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
max_query_len=max_query_len,
|
||||
max_seq_len=max_seq_len,
|
||||
softmax_scale=HEAD_DIM**-0.5,
|
||||
causal=False,
|
||||
)
|
||||
|
||||
new_k = torch.cat([history_k, current_k], dim=1)
|
||||
new_v = torch.cat([history_v, current_v], dim=1)
|
||||
if action_len:
|
||||
new_k = torch.cat([new_k, action_k], dim=1)
|
||||
new_v = torch.cat([new_v, action_v], dim=1)
|
||||
ref = _dense_attention(query, new_k, new_v)
|
||||
|
||||
torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
def test_a_ragged_scratch_chunk_is_physically_continuous_with_its_history():
|
||||
"""The write targets themselves must leave no hole.
|
||||
|
||||
This is the property the values above depend on, asserted directly so a
|
||||
regression names the cause rather than showing a numeric mismatch.
|
||||
"""
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(device=device, window_chunks=2, chunk_size=RAGGED_CHUNK)
|
||||
_commit_video_span(kv, st, kv_branch=POS, n_chunks=1, dtype=torch.float32, device=device, chunk_size=RAGGED_CHUNK)
|
||||
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=False)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
|
||||
history_tail_block = ctx.history_block_ids[-1]
|
||||
# The chunk starts inside the history's own last block, not at a fresh one.
|
||||
assert ctx.current_video_block_ids[0] == history_tail_block
|
||||
first_slot = int(ctx.current_video_slot_mapping[0])
|
||||
assert first_slot == history_tail_block * BLOCK + 8
|
||||
|
||||
# And the action region must not be charged for that managed block.
|
||||
ctx.ensure_action_slots(3, device)
|
||||
assert ctx._scratch_blocks_used == len(ctx.current_video_block_ids) - 1
|
||||
assert ctx.action_scratch_block_ids[0] not in ctx.current_video_block_ids
|
||||
|
||||
|
||||
def test_a_ragged_window_keeps_the_block_table_a_fixed_shape():
|
||||
"""The table's shape must not track how full the window is.
|
||||
|
||||
With a sink the visible window is 2*24 + 24 = 72 tokens, which is 4.5
|
||||
blocks. Deriving the width by flooring that would understate the capacity,
|
||||
the max() against the live block count would take over, and the shape would
|
||||
change as the window filled -- recompiling the graph. The same floor would
|
||||
also let max_seq_len come out under the kv_len actually passed.
|
||||
"""
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(device=device, window_chunks=2, sink_chunks=1, chunk_size=RAGGED_CHUNK)
|
||||
# The window is deliberately not a whole number of blocks.
|
||||
max_video_tokens = kv.spec.sliding_window + kv.spec.sink_chunks * kv.spec.chunk_size
|
||||
assert max_video_tokens % BLOCK != 0
|
||||
|
||||
widths: set[int] = set()
|
||||
kv_lens: list[int] = []
|
||||
for _ in range(6):
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=True)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
block_table, _, seq_lens, _, max_seq_len = ctx.build_block_table(
|
||||
action_len=0, query_len=RAGGED_CHUNK, device=device
|
||||
)
|
||||
widths.add(int(block_table.shape[1]))
|
||||
# A bound handed to the kernel has to actually bound the sequence.
|
||||
assert int(seq_lens[0]) <= max_seq_len
|
||||
kv_lens.append(int(seq_lens[0]))
|
||||
st.commit_paged_context(POS)
|
||||
|
||||
assert len(widths) == 1, f"block table width varied across ticks: {sorted(widths)}"
|
||||
# The window has to have actually filled, or none of the above was tested.
|
||||
assert max(kv_lens) > min(kv_lens)
|
||||
assert max(kv_lens) <= ctx.max_video_blocks * BLOCK
|
||||
# Whole blocks are read, so each of the two boundaries -- the sink's end and
|
||||
# the recent window's start -- can contribute up to BLOCK - 1 tokens the
|
||||
# window no longer needs.
|
||||
assert max(kv_lens) <= max_video_tokens + 2 * (BLOCK - 1)
|
||||
|
||||
|
||||
def _poison_pools(kv: ARDiffusionKVCache) -> None:
|
||||
for pool in (*kv._k_pools, *kv._v_pools):
|
||||
pool.fill_(float("nan"))
|
||||
|
||||
|
||||
def _write_prepared_chunk(
|
||||
kv: ARDiffusionKVCache, ctx, *, dtype: torch.dtype, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Write fresh K/V for a prepared chunk, poisoning the rest of its blocks first.
|
||||
|
||||
Blocks are handed out whole and a reused block keeps whatever it held, so a
|
||||
slot read without having been written would not look unusual. Every slot of
|
||||
the chunk's blocks that this chunk does not write is set to NaN first --
|
||||
except the leading slots of a block the history still occupies -- so reading
|
||||
one turns the attention output NaN.
|
||||
"""
|
||||
mapping = ctx.current_video_slot_mapping
|
||||
first_slot = int(mapping[0])
|
||||
for index, block in enumerate(ctx.current_video_block_ids):
|
||||
start = first_slot if index == 0 else block * BLOCK
|
||||
kv._k_pools[0][start : (block + 1) * BLOCK] = float("nan")
|
||||
kv._v_pools[0][start : (block + 1) * BLOCK] = float("nan")
|
||||
k = torch.randn(ctx.seq_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
v = torch.randn(ctx.seq_len, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv._k_pools[0][mapping] = k
|
||||
kv._v_pools[0][mapping] = v
|
||||
return k, v
|
||||
|
||||
|
||||
def _window_positions(end: int, *, sink: int, window: int, written, history: int, resident: set[int]) -> list[int]:
|
||||
"""Token positions attention must read at ``end``.
|
||||
|
||||
Every token of the current chunk, plus every written history token in a
|
||||
block that is still resident and holds a sink token or one of the window's
|
||||
most recent tokens.
|
||||
"""
|
||||
start = end - window
|
||||
blocks = {p // BLOCK for p in range(min(sink, end))} | {p // BLOCK for p in range(max(start, 0), end)}
|
||||
return [p for p in sorted(written) if p >= history or (p // BLOCK in blocks and p // BLOCK in resident)]
|
||||
|
||||
|
||||
def _model_block_indices(kv, adapter) -> dict[int, int]:
|
||||
"""Each resident block's index in model positions, by block id.
|
||||
|
||||
Compaction removes the evicted gap after the sink's blocks and shifts the
|
||||
rest of the table down by it, so a table index past the sink is
|
||||
``compacted_tokens`` behind the position it holds.
|
||||
"""
|
||||
block_size = kv.block_size
|
||||
sink_blocks = -(-(kv.spec.sink_chunks * kv.spec.chunk_size) // block_size)
|
||||
shift = adapter.compacted_tokens // block_size
|
||||
return {
|
||||
int(block): index if index < sink_blocks else index + shift
|
||||
for index, block in enumerate(kv.block_table(adapter))
|
||||
if block != kv.null_block_id
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sink_chunks", "window_chunks", "reset_at_boundary"),
|
||||
[(1, 2, False), (0, 2, False), (1, 3, False), (2, 2, True), (0, 2, True)],
|
||||
)
|
||||
def test_attention_after_eviction_reads_every_kept_token_and_nothing_unwritten(
|
||||
sink_chunks, window_chunks, reset_at_boundary
|
||||
):
|
||||
"""Once the window slides, the table must still cover the window exactly.
|
||||
|
||||
The sink and the recent window are separate token ranges, and each can
|
||||
straddle a block edge on its own: 24-token chunks against 16-token blocks
|
||||
put a boundary 8 tokens past an edge on alternating chunks. Counting blocks
|
||||
off the ends of the resident list dropped the block holding the window's
|
||||
first tokens whenever both boundaries straddled, and the returned length
|
||||
ran past the last written slot. Neither changes a shape, so this compares
|
||||
values against dense attention over the tokens the window keeps -- after
|
||||
eviction, with every unwritten slot poisoned, on both the scratch and the
|
||||
committing forward of every chunk.
|
||||
|
||||
The reset case keeps only the sink across a boundary, so the recent window
|
||||
reaches back over blocks that are no longer resident. Those must be skipped
|
||||
by position rather than read.
|
||||
|
||||
Eight chunks are enough for compaction to drop the evicted gap from the
|
||||
table, after which storage positions run behind model positions. Removing
|
||||
a gap that is not a whole number of chunks moves where eviction snaps, and
|
||||
in the reset case the window then keeps tokens a boundary should have
|
||||
dropped.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device, dtype = torch.device("cpu"), torch.float32
|
||||
kv, st = make_state(
|
||||
device=device,
|
||||
window_chunks=window_chunks,
|
||||
sink_chunks=sink_chunks,
|
||||
reset_at_boundary=reset_at_boundary,
|
||||
chunk_size=RAGGED_CHUNK,
|
||||
)
|
||||
_poison_pools(kv)
|
||||
sink, window = sink_chunks * RAGGED_CHUNK, int(kv.spec.sliding_window)
|
||||
committed: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
|
||||
evicted = compacted = False
|
||||
|
||||
for chunk in range(8):
|
||||
history = int(st.adapter(POS).absolute_num_computed_tokens)
|
||||
end = history + RAGGED_CHUNK
|
||||
for commit_current in (False, True):
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=commit_current)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
k, v = _write_prepared_chunk(kv, ctx, dtype=dtype, device=device)
|
||||
written = dict(committed)
|
||||
written.update({history + i: (k[i], v[i]) for i in range(RAGGED_CHUNK)})
|
||||
|
||||
query = torch.randn(1, RAGGED_CHUNK, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
block_table, query_start_loc, seq_lens, max_query_len, max_seq_len = ctx.build_block_table(
|
||||
action_len=0, query_len=RAGGED_CHUNK, device=device
|
||||
)
|
||||
resident = set(_model_block_indices(kv, st.adapter(POS)).values())
|
||||
positions = _window_positions(
|
||||
end, sink=sink, window=window, written=written, history=history, resident=resident
|
||||
)
|
||||
label = f"chunk {chunk}, commit_current={commit_current}"
|
||||
assert int(seq_lens[0]) == len(positions), f"{label}: kv_len does not match the written slots it covers"
|
||||
paged = ar_diffusion_paged_attention(
|
||||
query,
|
||||
kv.key_cache(0),
|
||||
kv.value_cache(0),
|
||||
block_table=block_table,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
max_query_len=max_query_len,
|
||||
max_seq_len=max_seq_len,
|
||||
softmax_scale=HEAD_DIM**-0.5,
|
||||
causal=False,
|
||||
)
|
||||
ref = _dense_attention(
|
||||
query,
|
||||
torch.stack([written[p][0] for p in positions]).unsqueeze(0),
|
||||
torch.stack([written[p][1] for p in positions]).unsqueeze(0),
|
||||
)
|
||||
torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5, msg=lambda m, label=label: f"{label}: {m}")
|
||||
committed = written
|
||||
st.commit_paged_context(POS)
|
||||
compacted = compacted or st.adapter(POS).compacted_tokens > 0
|
||||
evicted = evicted or compacted or kv.null_block_id in kv.block_table(st.adapter(POS))
|
||||
|
||||
assert evicted, "the window never slid, so nothing after eviction was tested"
|
||||
assert compacted, "the table was never compacted, so storage and model positions never diverged"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sink_chunks", "window_chunks", "reset_at_boundary"),
|
||||
[(1, 2, False), (0, 2, False), (1, 3, False), (2, 2, True), (0, 2, True)],
|
||||
)
|
||||
def test_compaction_does_not_change_which_tokens_stay_resident(
|
||||
monkeypatch, sink_chunks, window_chunks, reset_at_boundary
|
||||
):
|
||||
"""Compaction renumbers the block table; it must not change what eviction keeps.
|
||||
|
||||
Storage positions run ``compacted_tokens`` behind model positions, and
|
||||
eviction snaps to chunk boundaries counted in storage positions. Removing a
|
||||
gap that is not a whole number of chunks moves every later snap, so eviction
|
||||
keeps a different set of tokens -- in the reset case, tokens a boundary
|
||||
should have dropped, which the window then reads. Attention checked against
|
||||
what is resident cannot see that, so the oracle here is the same rollout
|
||||
with compaction switched off.
|
||||
"""
|
||||
|
||||
def resident_blocks_per_tick(compact: bool) -> tuple[list[set[int]], int]:
|
||||
with monkeypatch.context() as patch:
|
||||
if not compact:
|
||||
patch.setattr(ChunkWindowManager, "compact_block_table", lambda self, request_id: 0)
|
||||
kv, st = make_state(
|
||||
window_chunks=window_chunks,
|
||||
sink_chunks=sink_chunks,
|
||||
reset_at_boundary=reset_at_boundary,
|
||||
chunk_size=RAGGED_CHUNK,
|
||||
)
|
||||
ticks = []
|
||||
for _ in range(12):
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=True)[0].forward_ctx
|
||||
ctx.ensure_video_slots(torch.device("cpu"))
|
||||
ticks.append(set(_model_block_indices(kv, st.adapter(POS)).values()))
|
||||
st.commit_paged_context(POS)
|
||||
return ticks, st.adapter(POS).compacted_tokens
|
||||
|
||||
expected, _ = resident_blocks_per_tick(compact=False)
|
||||
actual, compacted_tokens = resident_blocks_per_tick(compact=True)
|
||||
assert compacted_tokens > 0, "the table was never compacted, so nothing was tested"
|
||||
for tick, (want, got) in enumerate(zip(expected, actual)):
|
||||
assert got == want, f"tick {tick}: compaction kept blocks {sorted(got - want)} and dropped {sorted(want - got)}"
|
||||
|
||||
|
||||
def test_history_staging_holds_a_ragged_window_and_restages_it_whole(monkeypatch):
|
||||
"""The staging buffers must fit the table width this module pads to.
|
||||
|
||||
A sink or window that is not a whole number of blocks spans one more block
|
||||
than its token count suggests, so sizing staging from tokens left it short
|
||||
and the first prepared forward refused to run. And because such a chunk
|
||||
shares its first block with the history's tail, the history a probe sees is
|
||||
not left untouched by the next probe, so it is restaged rather than reused.
|
||||
"""
|
||||
monkeypatch.setenv(KV_GATHER_ENV, "1")
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(
|
||||
device=device, window_chunks=2, sink_chunks=1, chunk_size=RAGGED_CHUNK, reuse_history_staging=True
|
||||
)
|
||||
capacity = int(kv.history_staging[0][0].shape[0])
|
||||
for chunk in range(6):
|
||||
for probe in range(2):
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=False)[0].forward_ctx
|
||||
ctx.prepare(device=device, action_len=0, query_len=RAGGED_CHUNK)
|
||||
assert int(ctx.max_seq_len) <= capacity
|
||||
assert not ctx.reuse_history, f"chunk {chunk}, probe {probe}: a ragged chunk reused the staged history"
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=True)[0].forward_ctx
|
||||
ctx.prepare(device=device, action_len=0, query_len=RAGGED_CHUNK)
|
||||
st.commit_paged_context(POS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("commit_current", [False, True])
|
||||
def test_action_tokens_after_a_partly_written_video_block_are_refused(commit_current):
|
||||
"""Action K/V follows the video blocks in the table, and the kernel reads it as one run.
|
||||
|
||||
A 24-token chunk from an empty history ends 8 slots into its second 16-token
|
||||
block, so the run would read those 8 unwritten slots as the first action
|
||||
tokens and never reach the last ones. Every shape stays right, which is why
|
||||
this is refused rather than computed.
|
||||
"""
|
||||
_, st = make_state(window_chunks=2, chunk_size=RAGGED_CHUNK)
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=commit_current)[0].forward_ctx
|
||||
with pytest.raises(ValueError, match="partly written video block"):
|
||||
ctx.build_block_table(action_len=3, query_len=RAGGED_CHUNK + 3, device=torch.device("cpu"))
|
||||
|
||||
|
||||
def test_the_shipped_geometry_reads_exactly_the_window_on_every_tick():
|
||||
"""832x480 with the checkpoint's own sink and window, slot by slot.
|
||||
|
||||
A 9-frame sink and a 9-frame window are each 8 tokens past a 16-token edge
|
||||
but 28080 tokens together -- a whole number of blocks -- so a width taken
|
||||
from their sum was one block short on every tick once the window had
|
||||
filled. Twelve ticks of three frames, checking every slot the kernel reads
|
||||
against the token that slot is supposed to hold. No attention is computed:
|
||||
at this size the bookkeeping is the whole question.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
frame, frames_per_tick = (480 // 16) * (832 // 16), 3
|
||||
block = paging_block_size(frame)
|
||||
seq_len = frames_per_tick * frame
|
||||
cfg = ARDiffusionKVConfig(enable=True, chunk_size=frame, window_chunks=9, sink_chunks=9)
|
||||
kv = ARDiffusionKVCache(
|
||||
cfg,
|
||||
num_layers=1,
|
||||
num_kv_heads=1,
|
||||
head_size=8,
|
||||
dtype=torch.float16,
|
||||
block_size=block,
|
||||
max_model_len=1 << 20,
|
||||
available_bytes=1 << 27,
|
||||
kv_branches=(ARDiffusionKVBranchSpec(POS, 0),),
|
||||
session_capacity=1,
|
||||
frames_per_block=frames_per_tick,
|
||||
max_scratch_tokens_per_branch=seq_len,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
st = ARDiffusionKVState(kv, "s1", {POS: kv.begin_request("r-pos")}, num_layers=1)
|
||||
sink, window = 9 * frame, int(kv.spec.sliding_window)
|
||||
holds: dict[int, int] = {} # slot -> token position last written there
|
||||
widths: set[int] = set()
|
||||
filled = compacted = False
|
||||
|
||||
for tick in range(12):
|
||||
history = int(st.adapter(POS).absolute_num_computed_tokens)
|
||||
end = history + seq_len
|
||||
ctx = st.get_kv_caches(POS, seq_len=seq_len, commit_current=True)[0].forward_ctx
|
||||
ctx.ensure_video_slots(torch.device("cpu"))
|
||||
for offset, slot in enumerate(ctx.current_video_slot_mapping.tolist()):
|
||||
holds[slot] = history + offset
|
||||
table, _, seq_lens, _, _ = ctx.build_block_table(action_len=0, query_len=seq_len, device=torch.device("cpu"))
|
||||
widths.add(int(table.shape[1]))
|
||||
kv_len = int(seq_lens[0])
|
||||
|
||||
position_of_block = {b: i * block for b, i in _model_block_indices(kv, st.adapter(POS)).items()}
|
||||
read = []
|
||||
for b in table[0].tolist()[: -(-kv_len // block)]:
|
||||
assert b in position_of_block, f"tick {tick}: read block {b}, which holds no token of this session"
|
||||
read.extend((b * block + o, position_of_block[b] + o) for o in range(block))
|
||||
read = read[:kv_len]
|
||||
wrong = [(slot, want) for slot, want in read if holds.get(slot) != want]
|
||||
assert not wrong, f"tick {tick}: {len(wrong)} slots read hold a token other than the one their position implies"
|
||||
|
||||
start = end - window
|
||||
kept = set(range(min(sink, end))) | set(range(max(start, 0), end))
|
||||
missed = kept - {want for _, want in read}
|
||||
assert not missed, f"tick {tick}: {len(missed)} tokens the window keeps were not read"
|
||||
|
||||
filled = filled or end > sink + window
|
||||
st.commit_paged_context(POS)
|
||||
compacted = compacted or st.adapter(POS).compacted_tokens > 0
|
||||
|
||||
assert filled, "the window never filled, so the case that fails was never reached"
|
||||
assert compacted, "the table was never compacted, so storage and model positions never diverged"
|
||||
assert len(widths) == 1, f"block table width varied across ticks: {sorted(widths)}"
|
||||
|
||||
|
||||
class _CountedBlockIds:
|
||||
"""Wraps a vLLM block manager's ``get_blocks`` and counts the block ids read.
|
||||
|
||||
Every path to a request's block ids goes through ``get_blocks``, so a copy of
|
||||
the whole table is counted in full even if only a few entries are indexed.
|
||||
"""
|
||||
|
||||
def __init__(self, manager):
|
||||
self.reads = 0
|
||||
self._get_blocks = manager.get_blocks
|
||||
|
||||
def __call__(self, request_id):
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
|
||||
blocks = self._get_blocks(request_id)
|
||||
return KVCacheBlocks(tuple([_CountedBlock(block, self) for block in group] for group in blocks.blocks))
|
||||
|
||||
|
||||
class _CountedBlock:
|
||||
def __init__(self, block, counter: _CountedBlockIds):
|
||||
self._block = block
|
||||
self._counter = counter
|
||||
|
||||
@property
|
||||
def block_id(self) -> int:
|
||||
self._counter.reads += 1
|
||||
return self._block.block_id
|
||||
|
||||
|
||||
def test_reading_the_window_touches_only_the_blocks_it_can_keep(monkeypatch):
|
||||
"""However long the block table is, reading the window must not cost more.
|
||||
|
||||
Evicted positions stay in the table as null entries until compaction drops
|
||||
them a whole number of chunks at a time -- up to two frames' worth of blocks
|
||||
at 832x480 -- so the table can still be far longer than the window. Only the
|
||||
sink's blocks and the recent window's blocks can hold a kept token, so those
|
||||
are all the read may touch. Compaction is switched off here so the table
|
||||
grows to many times the window, and reads are counted where the ids leave
|
||||
vLLM's block manager.
|
||||
"""
|
||||
monkeypatch.setattr(ChunkWindowManager, "compact_block_table", lambda self, request_id: 0)
|
||||
device = torch.device("cpu")
|
||||
kv, st = make_state(device=device, window_chunks=2, sink_chunks=1, chunk_size=RAGGED_CHUNK)
|
||||
reads: list[int] = []
|
||||
table_lengths: list[int] = []
|
||||
for _ in range(60):
|
||||
ctx = st.get_kv_caches(POS, seq_len=RAGGED_CHUNK, commit_current=True)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
counted = _CountedBlockIds(kv.manager)
|
||||
kv.manager.get_blocks = counted
|
||||
try:
|
||||
ctx.build_block_table(action_len=0, query_len=RAGGED_CHUNK, device=device)
|
||||
finally:
|
||||
del kv.manager.get_blocks
|
||||
reads.append(counted.reads)
|
||||
table_lengths.append(len(kv.block_table(st.adapter(POS))))
|
||||
st.commit_paged_context(POS)
|
||||
|
||||
assert table_lengths[-1] > 10 * ctx.max_video_blocks, "the table never outgrew the window, so nothing was tested"
|
||||
assert max(reads) <= ctx.max_video_blocks, (
|
||||
f"read {max(reads)} table entries for a {ctx.max_video_blocks}-block window"
|
||||
)
|
||||
|
||||
|
||||
def test_the_checkpoints_own_default_resolution_can_build_a_cache():
|
||||
"""832x480 is what LingBot World v2 ships as its default, and it could not run.
|
||||
|
||||
A frame-sized block made the resolution a kernel-compatibility question:
|
||||
1560 tokens per frame is not a multiple of 16, so FlashAttention's paged
|
||||
kernel rejected it and the default resolution had no realtime path at all.
|
||||
"""
|
||||
from vllm_omni.experimental.ar_diffusion.runner import paging_block_size
|
||||
|
||||
tokens_per_frame = (480 // 16) * (832 // 16)
|
||||
assert tokens_per_frame == 1560
|
||||
assert tokens_per_frame % 16 == 8, "the whole point is that a frame is not a legal block"
|
||||
|
||||
block_size = paging_block_size(tokens_per_frame)
|
||||
cfg = ARDiffusionKVConfig(enable=True, chunk_size=tokens_per_frame, window_chunks=2)
|
||||
kv = ARDiffusionKVCache(
|
||||
cfg,
|
||||
num_layers=1,
|
||||
num_kv_heads=N_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.float32,
|
||||
block_size=block_size,
|
||||
max_model_len=1 << 16,
|
||||
available_bytes=1 << 28,
|
||||
kv_branches=(ARDiffusionKVBranchSpec(POS, 0),),
|
||||
session_capacity=1,
|
||||
frames_per_block=2,
|
||||
max_scratch_tokens_per_branch=block_size,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert kv.block_size % 16 == 0
|
||||
# The eviction unit is still the frame; only the paging unit changed.
|
||||
assert kv.spec.chunk_size == tokens_per_frame
|
||||
assert kv.blocks_per_frame == -(-tokens_per_frame // block_size) == 98
|
||||
|
||||
|
||||
def test_the_shipped_resolution_geometry_matches_dense_attention():
|
||||
"""The real number, not a scaled-down stand-in: 1560 tokens per frame.
|
||||
|
||||
This is the case the issue reported failing at ~4e-02 on the scratch path.
|
||||
It runs on CPU because the reference is dense attention, not a kernel.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cpu")
|
||||
dtype = torch.float32
|
||||
tokens_per_frame = (480 // 16) * (832 // 16)
|
||||
assert tokens_per_frame == 1560
|
||||
|
||||
kv, st = make_state(dtype=dtype, device=device, window_chunks=2, chunk_size=tokens_per_frame)
|
||||
history_k, history_v = _commit_video_span(
|
||||
kv, st, kv_branch=POS, n_chunks=1, dtype=dtype, device=device, chunk_size=tokens_per_frame
|
||||
)
|
||||
|
||||
# The non-committing path is the one that was wrong: four of the five
|
||||
# forwards per generated block take it.
|
||||
ctx = st.get_kv_caches(POS, seq_len=tokens_per_frame, commit_current=False)[0].forward_ctx
|
||||
ctx.ensure_video_slots(device)
|
||||
assert ctx.start_offset == 8, "1560 % 16 == 8 is the whole reason this case exists"
|
||||
|
||||
current_k = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
current_v = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv._k_pools[0][ctx.current_video_slot_mapping] = current_k[0]
|
||||
kv._v_pools[0][ctx.current_video_slot_mapping] = current_v[0]
|
||||
|
||||
query = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
block_table, query_start_loc, seq_lens, max_query_len, max_seq_len = ctx.build_block_table(
|
||||
action_len=0, query_len=tokens_per_frame, device=device
|
||||
)
|
||||
paged = ar_diffusion_paged_attention(
|
||||
query,
|
||||
kv.key_cache(0),
|
||||
kv.value_cache(0),
|
||||
block_table=block_table,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
max_query_len=max_query_len,
|
||||
max_seq_len=max_seq_len,
|
||||
softmax_scale=HEAD_DIM**-0.5,
|
||||
causal=False,
|
||||
)
|
||||
ref = _dense_attention(
|
||||
query,
|
||||
torch.cat([history_k, current_k], dim=1),
|
||||
torch.cat([history_v, current_v], dim=1),
|
||||
)
|
||||
torch.testing.assert_close(paged, ref, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
|
||||
@pytest.mark.parametrize("commit_current", [False, True])
|
||||
def test_the_shipped_resolution_geometry_on_the_real_kernel(commit_current):
|
||||
"""The shipped 832x480 geometry through FlashAttention's paged kernel.
|
||||
|
||||
The CPU test above proves the addressing is right against a dense
|
||||
reference. This one proves the real kernel agrees, at the real 1560
|
||||
tokens per frame, through the production path -- host prep followed by
|
||||
the fused write+attend op, which is what a forward actually calls.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.float16
|
||||
tokens_per_frame = (480 // 16) * (832 // 16)
|
||||
assert tokens_per_frame == 1560
|
||||
|
||||
kv, st = make_state(dtype=dtype, device=device, window_chunks=2, chunk_size=tokens_per_frame)
|
||||
history_k, history_v = _commit_video_span(
|
||||
kv, st, kv_branch=POS, n_chunks=1, dtype=dtype, device=device, chunk_size=tokens_per_frame
|
||||
)
|
||||
|
||||
layer_ctx = st.get_kv_caches(POS, seq_len=tokens_per_frame, commit_current=commit_current)[0]
|
||||
ctx = layer_ctx.forward_ctx
|
||||
current_k = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
current_v = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
query = torch.randn(1, tokens_per_frame, N_HEADS, HEAD_DIM, dtype=dtype, device=device)
|
||||
|
||||
ctx.prepare(device=device, action_len=0, query_len=tokens_per_frame)
|
||||
assert ctx.start_offset == 8, "1560 % 16 == 8 is the whole reason this case exists"
|
||||
assert kv.block_size == 16
|
||||
|
||||
inputs = layer_ctx.to_layer_inputs()
|
||||
paged = paged_write_attn(inputs, query[0], current_k[0], current_v[0], None, None, HEAD_DIM**-0.5).unsqueeze(0)
|
||||
|
||||
new_k = torch.cat([history_k, current_k], dim=1)[:, -kv.spec.sliding_window :]
|
||||
new_v = torch.cat([history_v, current_v], dim=1)[:, -kv.spec.sliding_window :]
|
||||
ref = _dense_attention(query, new_k, new_v)
|
||||
|
||||
torch.testing.assert_close(paged, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
|
||||
@pytest.mark.parametrize("history_chunks", [0, 1, 3])
|
||||
@pytest.mark.parametrize("action_len", [0, 3])
|
||||
|
||||
Reference in new issue
Block a user