[Diffusion][HunyuanImage3] Add request paged prefix caching (#7719)
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
1 parent
9a1a05facd
commit
205aa01c7b
49 files changed
+2889
-127
No files matched your search
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -154,6 +154,7 @@ class DiffusionKVModelRunnerBackend:
|
||||
|
||||
if attention_geometry is not None:
|
||||
num_heads, num_kv_heads, head_size = attention_geometry
|
||||
assert callable(set_attention_geometry)
|
||||
try:
|
||||
set_attention_geometry(
|
||||
num_heads=num_heads,
|
||||
@@ -396,6 +397,15 @@ class DiffusionKVModelRunnerBackend:
|
||||
for sequence in metadata.sequences:
|
||||
if type(sequence.sequence_id) is not int or sequence.sequence_id < 0:
|
||||
raise ValueError(f"Diffusion KV sequence_id must be a non-negative integer: {sequence.sequence_id!r}")
|
||||
if (
|
||||
type(sequence.cached_prefix_len) is not int
|
||||
or sequence.cached_prefix_len < 0
|
||||
or sequence.cached_prefix_len > sequence.prefix_len
|
||||
):
|
||||
raise ValueError(
|
||||
"Diffusion KV cached_prefix_len must lie within the stable prefix: "
|
||||
f"cached={sequence.cached_prefix_len!r}, prefix={sequence.prefix_len!r}"
|
||||
)
|
||||
sequence_identity = (metadata.request_id, sequence.sequence_id, None)
|
||||
sequence_install = self._validate_row(
|
||||
identity=sequence_identity,
|
||||
@@ -410,6 +420,7 @@ class DiffusionKVModelRunnerBackend:
|
||||
sequence.prefix_len,
|
||||
sequence.target_len,
|
||||
sequence.seq_len,
|
||||
sequence.cached_prefix_len,
|
||||
sequence_install.block_ids,
|
||||
sequence.context_ids,
|
||||
)
|
||||
@@ -593,7 +604,7 @@ class DiffusionKVModelRunnerBackend:
|
||||
)
|
||||
raise RuntimeError(f"Diffusion KV request state is missing logical length for {identity!r}")
|
||||
|
||||
def remove_diffusion_kv_requests(self, request_ids: list[str | tuple[str, int]]) -> int:
|
||||
def remove_diffusion_kv_requests(self, request_ids: Sequence[str | tuple[str, int]]) -> int:
|
||||
"""Retire Worker rows without logically freeing Scheduler-owned blocks."""
|
||||
if (
|
||||
not is_scheduler_paged_kv_mode(
|
||||
|
||||
Reference in new issue
Block a user