[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Fable 5.1 authored and GitHub committed 2026-09-21 06:12:45 +00:00
1 parent a9e2117f5e
commit 86fcfe95f1
23 files changed
+1957 -200

No files matched your search

+91 -2
View File
@@ -14,13 +14,14 @@ from vllm.logger import init_logger
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.models.interface import supports_step_execution
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPrefetchJob
from vllm_omni.diffusion.sched.interface import CachedRequestData, DiffusionSchedulerOutput, KVPrefetchJob
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput
from vllm_omni.experimental.ar_diffusion.capability import (
ARDiffusionKVCacheSpec,
SupportsARDiffusionPipeline,
SupportsARDiffusionWarmup,
supports_chunk_step_grouping,
)
from vllm_omni.experimental.ar_diffusion.kv_cache.config import ARDiffusionKVConfig
from vllm_omni.experimental.ar_diffusion.kv_cache.manager import ARDiffusionKVCache
@@ -144,6 +145,7 @@ class ARDiffusionModelRunner(DiffusionModelRunner):
kv_branches=spec.kv_branches,
session_capacity=spec.session_capacity,
cross_attention_lengths=spec.cross_attention_lengths,
cross_attention_kv_heads=spec.cross_attention_kv_heads,
frames_per_block=spec.frames_per_block,
max_scratch_tokens_per_branch=spec.max_scratch_tokens_per_branch,
model_owned_state_bytes_per_session=spec.model_owned_state_bytes_per_session,
@@ -163,7 +165,10 @@ class ARDiffusionModelRunner(DiffusionModelRunner):
config.window_chunks,
config.sink_chunks,
[(kv_branch.name, kv_branch.local_index) for kv_branch in spec.kv_branches],
spec.cross_attention_lengths,
{
name: (length, spec.cross_attention_kv_heads.get(name, spec.num_kv_heads))
for name, length in spec.cross_attention_lengths.items()
},
self._session_capacity,
spec.session_capacity,
)
@@ -338,6 +343,90 @@ class ARDiffusionModelRunner(DiffusionModelRunner):
"ARDiffusionModelRunner does not support request-batch execution; use request mode with max_num_seqs=1."
)
def _groups_chunk_steps(self, scheduler_output: DiffusionSchedulerOutput) -> bool:
"""Policy: does this stepwise call run the scheduled request's chunk to its boundary?
Only on the realtime AR path, for a lone streaming request of a
pipeline that declares ``supports_chunk_step_grouping``. The session's
KV is bound for the whole call and the scheduler regains control at
every chunk boundary, which is where this path's interactions are
applied anyway; what it gives up is the chance to act between two
steps of one chunk.
"""
return (
bool(self.od_config.streaming_output)
and len(scheduler_output.scheduled_request_ids) == 1
and supports_chunk_step_grouping(self.pipeline)
)
def _execute_stepwise_core(
self,
scheduler_output: DiffusionSchedulerOutput,
*,
record_output_peak_memory: bool,
in_diffusion_kv_memory_profile: bool = False,
) -> BatchRunnerOutput:
"""Run the shared single-step core once, or repeatedly until the chunk boundary.
The shared runner keeps its one-step-per-call semantics. When the
grouping policy applies, the remaining steps of the request's current
chunk are driven from here with a continuation of the scheduler
output: the request is presented as a cached request, with nothing
new to admit, nothing to retire and no connector work, so every
further step is exactly what the next scheduler cycle would have
asked for. Grouping the probes and the commit of one block this way
drops the scheduler/executor round trips between them.
The loop stops at the first call that emits a result (a chunk or an
error), finishes the request, or leaves it without state; the base
core already handles a mid-chunk failure or interrupt by finishing
the request with an error and dropping its state.
"""
output = super()._execute_stepwise_core(
scheduler_output,
record_output_peak_memory=record_output_peak_memory,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
)
if not self._groups_chunk_steps(scheduler_output):
return output
request_id = scheduler_output.scheduled_request_ids[0]
continuation = dataclasses.replace(
scheduler_output,
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData(request_ids=[request_id]),
finished_req_ids=set(),
kv_prefetch_job=None,
kv_connector_metadata=None,
)
if not self._chunk_step_pending(output, request_id):
return output
max_steps = self.state_cache[request_id].chunk_num_steps
if max_steps is None:
return output
steps = 1
while self._chunk_step_pending(output, request_id):
if steps >= max_steps:
self.state_cache.pop(request_id)
runner_output = output.get_request_output(request_id)
assert runner_output is not None
runner_output.finished = True
runner_output.result = DiffusionOutput(error=f"Chunk did not complete within {max_steps} denoise steps")
break
steps += 1
output = super()._execute_stepwise_core(
continuation,
record_output_peak_memory=record_output_peak_memory,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
)
return output
def _chunk_step_pending(self, output: BatchRunnerOutput, request_id: str) -> bool:
"""More steps of the request's current chunk remain after ``output``."""
runner_output = output.get_request_output(request_id)
if runner_output is None or runner_output.finished or runner_output.result is not None:
return False
return request_id in self.state_cache
def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput:
"""Bind runner-owned KV for one stepwise invocation, then inherit the step loop."""
pipeline = getattr(self, "pipeline", None)