rebase: align vllm-omni with vLLM e9163b536e72
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
f137a1e3ad
commit
3a508d7b7a
14 files changed
+97
-46
No files matched your search
@@ -87,7 +87,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
if self.execute_model_state is not None:
|
||||
raise RuntimeError("State error: sample_tokens() must be called after execute_model() returns None.")
|
||||
|
||||
if self.vllm_config.model_config.enable_return_routed_experts:
|
||||
if self.routed_experts_initialized:
|
||||
capturer = RoutedExpertsCapturer.get_instance()
|
||||
if capturer is not None:
|
||||
capturer.clear_buffer() # noqa
|
||||
@@ -265,9 +265,9 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
# When spec decode is enabled, defer clearing KV connector metadata
|
||||
# until after draft model runs in sample_tokens.
|
||||
clear_kv_metadata = self.speculative_config is None
|
||||
# When spec decode is enabled, defer connector finalization
|
||||
# (wait_for_save + clear metadata) until after draft model runs.
|
||||
defer_kv_connector_finalize = self.speculative_config is not None
|
||||
with (
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
@@ -282,7 +282,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
record_function_or_nullcontext("Forward"),
|
||||
self.maybe_get_kv_connector_output(
|
||||
scheduler_output,
|
||||
clear_metadata=clear_kv_metadata,
|
||||
defer_finalize=defer_kv_connector_finalize,
|
||||
) as kv_connector_output,
|
||||
):
|
||||
outputs = self._run_generation_model(
|
||||
|
||||
Reference in new issue
Block a user