[Diffusion][HunyuanImage3] Add request paged prefix caching (#7719)
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
1 parent
9a1a05facd
commit
205aa01c7b
49 files changed
+2889
-127
No files matched your search
@@ -38,7 +38,7 @@ from vllm_omni.diffusion.sched.interface import (
|
||||
StepBatchSamplingParamsKey,
|
||||
_AdmissionWaitDecision,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import RunnerOutput
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -120,6 +120,12 @@ class BaseScheduler(ABC):
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
max_in_flight_tokens=kv_vllm_config.max_in_flight_tokens,
|
||||
enable_prefix_caching=bool(getattr(kv_vllm_config.cache_config, "enable_prefix_caching", False)),
|
||||
prefix_caching_hash_algo=getattr(
|
||||
kv_vllm_config.cache_config,
|
||||
"prefix_caching_hash_algo",
|
||||
"sha256",
|
||||
),
|
||||
)
|
||||
else:
|
||||
if any(
|
||||
@@ -182,7 +188,8 @@ class BaseScheduler(ABC):
|
||||
|
||||
diffusion_kv_metadata: DiffusionKVMetadata | None = None
|
||||
if self._diffusion_kv_manager is not None:
|
||||
if self._diffusion_kv_manager.has_request(request_id):
|
||||
already_reserved = self._diffusion_kv_manager.has_request(request_id)
|
||||
if already_reserved:
|
||||
diffusion_kv_metadata = self._diffusion_kv_manager.get_metadata(request_id)
|
||||
else:
|
||||
matched_tokens: list[int] = []
|
||||
@@ -210,6 +217,11 @@ class BaseScheduler(ABC):
|
||||
continue
|
||||
if allocation is None:
|
||||
break
|
||||
# Check the lookup result before registering destination
|
||||
# pages with the connector: deferred pages can be freed.
|
||||
if not self._can_schedule_waiting(state):
|
||||
self._diffusion_kv_manager.free_request(request_id)
|
||||
break
|
||||
diffusion_kv_metadata = allocation
|
||||
self._kv_request_generations[request_id] = allocation.allocation_generation
|
||||
if self._kv_connector is not None:
|
||||
@@ -397,7 +409,7 @@ class BaseScheduler(ABC):
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
|
||||
pass
|
||||
|
||||
def has_requests(self) -> bool:
|
||||
@@ -633,6 +645,11 @@ class BaseScheduler(ABC):
|
||||
# Also surface admission failures recorded while schedule() built this
|
||||
# output. Older finished ids retained only for Worker cleanup have
|
||||
# already been popped by the Engine and are deliberately ignored.
|
||||
if self._diffusion_kv_manager is not None:
|
||||
for request_id, status in statuses.items():
|
||||
if status == DiffusionRequestStatus.FINISHED_COMPLETED:
|
||||
self._diffusion_kv_manager.publish_request(request_id)
|
||||
|
||||
finished_req_ids = {
|
||||
request_id for request_id in sched_output.finished_req_ids if request_id in self._request_states
|
||||
}
|
||||
|
||||
@@ -12,11 +12,12 @@ from vllm_omni.diffusion.sched.interface import (
|
||||
DiffusionRequestStatus,
|
||||
DiffusionSchedulerOutput,
|
||||
RequestBatchSamplingParamsKey,
|
||||
SchedulerRequestState,
|
||||
_AdmissionWaitDecision,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.worker.utils import RunnerOutput
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput
|
||||
|
||||
# LoRA identity is derived from `sampling.lora_request`, not a same-named field
|
||||
# on sampling params, so it must be resolved separately from the bulk lookup.
|
||||
@@ -32,7 +33,7 @@ def _normalize_explicit_sample_solver(value: object | None) -> str | None:
|
||||
return str(value).strip().lower()
|
||||
|
||||
|
||||
def _normalize_explicit_flow_shift(value: object | None) -> float | None:
|
||||
def _normalize_explicit_flow_shift(value: float | str | None) -> float | None:
|
||||
"""Normalize an explicitly provided flow shift without selecting a default."""
|
||||
if value is None:
|
||||
return None
|
||||
@@ -59,6 +60,18 @@ def build_request_batch_sampling_params_key(request: OmniDiffusionRequest) -> Re
|
||||
class RequestScheduler(BaseScheduler):
|
||||
"""Scheduler for static request waves, including admission coalescing."""
|
||||
|
||||
def _can_schedule_waiting(self, state: SchedulerRequestState) -> bool:
|
||||
if not super()._can_schedule_waiting(state):
|
||||
return False
|
||||
manager = self._diffusion_kv_manager
|
||||
if not self._running or manager is None or not manager.has_request(state.request_id):
|
||||
return True
|
||||
# The request-level runner uses one first-step query boundary per wave.
|
||||
# The Manager already aligns all CFG sequences within each request.
|
||||
current = manager.get_metadata(self._running[0]).sequences[0].cached_prefix_len
|
||||
candidate = manager.get_metadata(state.request_id).sequences[0].cached_prefix_len
|
||||
return candidate == current
|
||||
|
||||
def get_admission_wait_decision(
|
||||
self,
|
||||
*,
|
||||
@@ -98,7 +111,7 @@ class RequestScheduler(BaseScheduler):
|
||||
def _build_sampling_params_key(self, request: OmniDiffusionRequest) -> RequestBatchSamplingParamsKey:
|
||||
return build_request_batch_sampling_params_key(request)
|
||||
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
|
||||
scheduled_request_ids = sched_output.scheduled_request_ids
|
||||
if not scheduled_request_ids and not sched_output.finished_req_ids:
|
||||
return set()
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm_omni.diffusion.sched.interface import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.worker.utils import RunnerOutput
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -62,7 +62,7 @@ class StepScheduler(BaseScheduler):
|
||||
)
|
||||
return request_id
|
||||
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
|
||||
scheduled_request_ids = sched_output.scheduled_request_ids
|
||||
if not scheduled_request_ids and not sched_output.finished_req_ids:
|
||||
return set()
|
||||
@@ -125,6 +125,7 @@ class StepScheduler(BaseScheduler):
|
||||
return self._sequence_length(sampling.timesteps)
|
||||
if sampling.sigmas is not None:
|
||||
return len(sampling.sigmas)
|
||||
assert sampling.num_inference_steps is not None
|
||||
return int(sampling.num_inference_steps)
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in new issue
Block a user