[Bugfix] Fix DFlash prefix cache corruption due to missing lookahead block (#4449)
Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
1 parent
5f5b77aa9c
commit
864d22164c
1 file changed
+1
-4
@@ -1052,10 +1052,7 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
spec_config = self.speculative_config
|
||||
propose_drafts_after_bookkeeping = False
|
||||
if spec_config is not None:
|
||||
input_fits_in_drafter = spec_decode_common_attn_metadata is not None and (
|
||||
spec_decode_common_attn_metadata.max_seq_len + self.num_spec_tokens
|
||||
<= self.effective_drafter_max_model_len
|
||||
)
|
||||
input_fits_in_drafter = self._input_fits_in_drafter(spec_decode_common_attn_metadata)
|
||||
use_gpu_toks = (
|
||||
spec_config.use_eagle() or spec_config.uses_draft_model() or spec_config.uses_extract_hidden_states()
|
||||
) and not spec_config.disable_padded_drafter_batch
|
||||
|
||||
Reference in new issue
Block a user