[Feature] Support online inference (#64)

Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
Gao Han authored and GitHub committed 2025-11-17 14:44:10 +08:00
1 parent bf82e18374
commit c5a89302f1
52 files changed
+3666 -685

No files matched your search

+10 -7
View File
@@ -501,13 +501,16 @@ class GPUDiffusionModelRunner(OmniGPUModelRunner):
else:
cudagraph_runtime_mode = _cg_mode
with self.maybe_randomize_inputs(input_ids), set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=num_tokens_after_padding,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=cudagraph_runtime_mode,
batch_descriptor=batch_descriptor,
with (
self.maybe_randomize_inputs(input_ids),
set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=num_tokens_after_padding,
num_tokens_across_dp=num_tokens_across_dp,
cudagraph_runtime_mode=cudagraph_runtime_mode,
batch_descriptor=batch_descriptor,
),
):
outputs = self.model(
input_ids=input_ids,