[Diffusion][HunyuanImage3] Add request paged prefix caching (#7719)
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
1 parent
9a1a05facd
commit
205aa01c7b
49 files changed
+2889
-127
No files matched your search
@@ -38,7 +38,7 @@ from vllm_omni.diffusion.sched.interface import (
|
||||
StepBatchSamplingParamsKey,
|
||||
_AdmissionWaitDecision,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import RunnerOutput
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -120,6 +120,12 @@ class BaseScheduler(ABC):
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
max_in_flight_tokens=kv_vllm_config.max_in_flight_tokens,
|
||||
enable_prefix_caching=bool(getattr(kv_vllm_config.cache_config, "enable_prefix_caching", False)),
|
||||
prefix_caching_hash_algo=getattr(
|
||||
kv_vllm_config.cache_config,
|
||||
"prefix_caching_hash_algo",
|
||||
"sha256",
|
||||
),
|
||||
)
|
||||
else:
|
||||
if any(
|
||||
@@ -182,7 +188,8 @@ class BaseScheduler(ABC):
|
||||
|
||||
diffusion_kv_metadata: DiffusionKVMetadata | None = None
|
||||
if self._diffusion_kv_manager is not None:
|
||||
if self._diffusion_kv_manager.has_request(request_id):
|
||||
already_reserved = self._diffusion_kv_manager.has_request(request_id)
|
||||
if already_reserved:
|
||||
diffusion_kv_metadata = self._diffusion_kv_manager.get_metadata(request_id)
|
||||
else:
|
||||
matched_tokens: list[int] = []
|
||||
@@ -210,6 +217,11 @@ class BaseScheduler(ABC):
|
||||
continue
|
||||
if allocation is None:
|
||||
break
|
||||
# Check the lookup result before registering destination
|
||||
# pages with the connector: deferred pages can be freed.
|
||||
if not self._can_schedule_waiting(state):
|
||||
self._diffusion_kv_manager.free_request(request_id)
|
||||
break
|
||||
diffusion_kv_metadata = allocation
|
||||
self._kv_request_generations[request_id] = allocation.allocation_generation
|
||||
if self._kv_connector is not None:
|
||||
@@ -397,7 +409,7 @@ class BaseScheduler(ABC):
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
|
||||
pass
|
||||
|
||||
def has_requests(self) -> bool:
|
||||
@@ -633,6 +645,11 @@ class BaseScheduler(ABC):
|
||||
# Also surface admission failures recorded while schedule() built this
|
||||
# output. Older finished ids retained only for Worker cleanup have
|
||||
# already been popped by the Engine and are deliberately ignored.
|
||||
if self._diffusion_kv_manager is not None:
|
||||
for request_id, status in statuses.items():
|
||||
if status == DiffusionRequestStatus.FINISHED_COMPLETED:
|
||||
self._diffusion_kv_manager.publish_request(request_id)
|
||||
|
||||
finished_req_ids = {
|
||||
request_id for request_id in sched_output.finished_req_ids if request_id in self._request_states
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user