[Feature] Support online inference (#64)
Signed-off-by: Gaohan123 <gaohan19@huawei.com>
This commit is contained in:
1 parent
bf82e18374
commit
c5a89302f1
52 files changed
+3666
-685
No files matched your search
@@ -501,13 +501,16 @@ class GPUDiffusionModelRunner(OmniGPUModelRunner):
|
||||
else:
|
||||
cudagraph_runtime_mode = _cg_mode
|
||||
|
||||
with self.maybe_randomize_inputs(input_ids), set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens_after_padding,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_descriptor,
|
||||
with (
|
||||
self.maybe_randomize_inputs(input_ids),
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_tokens_after_padding,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_descriptor,
|
||||
),
|
||||
):
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
|
||||
Reference in new issue
Block a user